Merge branch 'develop'

Author: howjmay

.gitignore

@@ -92,3 +92,14 @@ docs/html/
 
 # Python
 __pycache__/
+
+# Certificate for build system
+pem/*.inc
+
+# Legato app related files
+resolv.conf
+output_base/
+_build_endpoint/
+endpoint.*.update
+.repo
+legato/

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 # Contributing to Tangle-accelerator
 
 When contributing to this repository, please first discuss the change you wish to make via issue,
-email, or any other method with the owners of this repository before making a change. 
+email, or any other method with the owners of this repository before making a change.
 
 Please note we have a code of conduct, please follow it in all your interactions with the project.
 
 ## Pull Request Process
 
-1. Ensure any install or build dependencies are removed before the end of the layer when doing a 
-   build.
+1. Ensure any install or build dependencies are removed before the end of the layer when doing a build.
 2. Create a new branch for fixing or developing the changes.
 3. Run test with `bazel test //tests/...`, after finishing the changes.
-4. Update the README.md with details of changes to the interface, which includes new environment 
-   variables, exposed ports, useful file locations and container parameters.
+4. Update the README.md with details of changes to the interface, which includes new environment variables, exposed ports, useful file locations and container parameters.
 5. Run `hooks/formatter` before committing the changes.
 6. Rebase to the latest `develop` branch.
 
 ## Git Commit Message Guidelines
+
 Read this [blog article](https://chris.beams.io/posts/git-commit/) and [this article](https://www.conventionalcommits.org/en/v1.0.0-beta.2/) and follow the instructions in these articles.
 The subject line of git commit message should follow this pattern
 `<type>[optional scope]: <description>`
@@ -28,6 +27,21 @@ The `type` includes the following 5 words depending on the content of the commit
 * test
 * doc
 
+## Doxygen-friendly Representation
+
+Function parameters should be in the following style or [doxygen official example](https://www.doxygen.nl/manual/commands.html#cmdparam) which allows the generating doxygen documentation.
+
+```c
+/**
+ * Copies bytes from a source memory area to a destination memory area,
+ * where both areas may not overlap.
+ * @param[out] dest The memory area to copy to.
+ * @param[in]  src  The memory area to copy from.
+ * @param[in]  n    The number of bytes to copy
+ */
+void memcpy(void *dest, const void *src, size_t n);
+```
+
 ## Code of Conduct
 
 ### Our Pledge

Doxyfile

@@ -1,6 +1,6 @@
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = "tangle-accelerator"
-PROJECT_NUMBER         = 0.1.0
+PROJECT_NUMBER         = 0.9.2
 OUTPUT_DIRECTORY       = docs/
 OPTIMIZE_OUTPUT_FOR_C  = YES
 INPUT                  = . \
@@ -9,13 +9,19 @@ INPUT                  = . \
                         accelerator/core/request \
                         accelerator/core/response \
                         accelerator/core/serializer \
+                        endpoint \
+                        endpoint/endpointComp \
+                        endpoint/hal \
+                        endpoint \
                         utils \ 
+                        utils/cache \ 
                         common \
                         storage \
                         connectivity/mqtt \
                         connectivity/http
 FILE_PATTERNS          = *.h \
-                        *.md
+                        *.md \
+                        *.c
 EXAMPLE_PATH           = tests
 EXAMPLE_PATTERNS       = test_*
 USE_MDFILE_AS_MAINPAGE = README.md

Makefile

@@ -2,11 +2,59 @@ DCURL_DIR := third_party/dcurl
 DCURL_LIB := $(DCURL_DIR)/build/libdcurl.so
 MOSQUITTO_DIR := third_party/mosquitto
 MOSQUITTO_LIB := $(MOSQUITTO_DIR)/lib/libmosquitto.so.1
+PEM_DIR = pem
+# Default pem file. See pem/README.md for more information
+PEM := $(PEM_DIR)/cert.pem
+OUTPUT_BASE_DIR := output_base
+# Endpoint build target. The default intends to the platform of your development system.
+EP_TARGET := simulator
+# Build test suite
+TESTS := false
+# Enable endpoint HTTPS connection to tangle-accelerator. 
+# The endpoint uses HTTP connection to transmit encrypted data by default.
+# See "HTTPS Connection Support" in docs/endpoint.md for more information.
+ENFORCE_EP_HTTPS := false
+# The flags that will be preprocessed by mkapp program, part of Legato Application Framework. 
+# Eventually, the processed flags are passed as compiler-time options.
+LEGATO_FLAGS :=
 DEPS += $(DCURL_LIB)
 
-all: $(DEPS)
+# Determine to enable HTTPS connection
+ifeq ($(ENFORCE_EP_HTTPS), true)
+	LEGATO_FLAGS += -DENDPOINT_HTTPS
+endif
 
-.PHONY: $(DCURL_LIB) $(MOSQUITTO_LIB)
+# Determine to build test suite
+ifeq ($(TESTS), true)
+	LEGATO_FLAGS += -DENABLE_ENDPOINT_TEST
+endif
+
+# The tangle-acclerator host for endpoint to connect
+ifdef EP_TA_HOST
+	LEGATO_FLAGS += -DEP_TA_HOST=${EP_TA_HOST}
+endif
+# The tangle-acclerator port for endpoint to connect
+ifdef EP_TA_PORT
+	LEGATO_FLAGS += -DEP_TA_PORT=${EP_TA_PORT}
+endif
+# The ssl seed for endpoint (optional)
+ifdef EP_SSL_SEED
+	LEGATO_FLAGS += -DEP_SSL_SEED=${EP_SSL_SEED}
+endif
+
+# Pass target into endpoint build process
+LEGATO_FLAGS += -DEP_TARGET=$(EP_TARGET)
+
+# Prepend the "-C" flag at the beginging for passing cflags into mkapp
+LEGATO_FLAGS := $(foreach flags, $(LEGATO_FLAGS), -C $(flags))
+
+# Include the build command from the specific target
+include endpoint/platform/$(EP_TARGET)/build.mk
+export EP_TARGET
+
+all: $(DEPS) cert
+
+.PHONY: $(DCURL_LIB) $(MOSQUITTO_LIB) legato cert check_pem
 
 $(DCURL_LIB): $(DCURL_DIR)
 	git submodule update --init $^
@@ -15,12 +63,36 @@ $(DCURL_LIB): $(DCURL_DIR)
 	$(info Modify $^/build/local.mk for your environments.)
 	$(MAKE) -C $^ all
 
-MQTT: $(DCURL_LIB) $(MOSQUITTO_LIB)
+MQTT: $(DCURL_LIB) $(MOSQUITTO_LIB) cert
 $(MOSQUITTO_LIB): $(MOSQUITTO_DIR)
 	git submodule update --init $^
 	@echo
 	$(MAKE) -C $^ WITH_DOCS=no
 
+# Build endpoint Legato app
+legato: cert
+	# Fetch the required external source code
+	# FIXME: Use 'fetch' instead of 'build' to avoid extra building actions.
+	#        The 'build' option is for getting the header file like 'mam/mam/mam_endpoint_t_set.h',
+	#        which can not be downloaded when the 'fetch' option is used.
+	bazel --output_base=$(OUTPUT_BASE_DIR) build //endpoint:libendpoint.so
+	# Generate endpoint Legato app
+	$(call platform-build-command)
+
+cert: check_pem
+	@xxd -i $(PEM) > $(PEM_DIR)/ca_crt.inc
+	@sed -E \
+		-e 's/(unsigned char)(.*)(\[\])(.*)/echo "\1 ca_crt_pem\3\4"/ge' 	    \
+		-e 's/(unsigned int)(.*)(=)(.*)/echo "\1 ca_crt_pem_len \3\4"/ge' 	    \
+		 -e 's/^unsigned/static &/' \
+		 -e 's/(.*)(pem_len = )([0-9]+)(.*)/echo "\1\2$$((\3+1))\4"/ge' \
+		 -e 's/(0[xX][[[:xdigit:]]+)$$/\1, 0x0/g' \
+	     -i $(PEM_DIR)/ca_crt.inc
+check_pem:
+ifndef PEM
+	$(error PEM is not set)
+endif
+
 clean:
 	$(MAKE) -C $(DCURL_DIR) clean
 	$(MAKE) -C $(MOSQUITTO_DIR) clean

README.md

@@ -19,7 +19,7 @@ use scenarios such as MAM and [TangleID](https://tangleid.github.io/).
 
 At the moment, it is not feasible to host fully-functioned full nodes on Raspberry Pi class
 Arm devices, but Raspberry Pi 3 is known to be capable to execute `Tangle-accelerator`
-without problems. Since it is written in C/C++ with [entangled](https://github.com/iotaledger/entangled),
+without problems. Since it is written in C/C++ with [iota.c](https://github.com/iotaledger/iota.c),
 both footprint and startup time are behaved pretty well.
 
 ## Architecture
@@ -28,7 +28,7 @@ both footprint and startup time are behaved pretty well.
 
 ```
                 +-------------------------------------------+
-+----------+    |  +-----------------+       +-----------+  |       
++----------+    |  +-----------------+       +-----------+  |
 |          |    |  | Service         |       | Cache     |  |
 |  Client  <-----> |                 | <---> |           |  |
 |          |    |  | -Explorer       |       | -Trytes   |  |
@@ -37,7 +37,7 @@ both footprint and startup time are behaved pretty well.
                 |  | -Proxy          |       |           |  |
                 |  +-----------------+       +-----------+  |
                 |         ^                                 |
-                +---------|---------------------------------+     
+                +---------|---------------------------------+
                           v
                 +-------------------------------------------+  
                 | Full Node                                 |
@@ -52,7 +52,7 @@ both footprint and startup time are behaved pretty well.
 
 `Tangle-accelerator` helps to reattach pending transactions were attached from `Tangle-accelerator`.
 Reattachment increases chances of confirmation and prevents messages being pruned when full nodes perform snapshot.
-Clients should provide a unique ID as the identifier to each message and it's corresponding transaction hash since a new transaction hash will be generated after reattachment. 
+Clients should provide a unique ID as the identifier to each message and it's corresponding transaction hash since a new transaction hash will be generated after reattachment.
 
 `Tangle-accelerator` uses ScyllaDB to store each transaction's ID, hash and status(Pending or confirmed). `Tangle-accelerator` will periodically check the status of pending transactions and reattach transactions which have been pended too long. Confirmed transactions will be stored into permanodes.
 
@@ -61,20 +61,23 @@ Clients can find the transaction alone with wanted message by using the ID to qu
 ## Connectivity
 
 `Tangle-accelerator`, at this moment, supports the following TCP/IP derived protocols:
+
 * `HTTP`
-* `MQTT` 
+* `MQTT`
 
 ### HTTP
+
 `HTTP` can be used in the normal internet service. User can use RESTful APIs to interact with `tangle-accelerator`.
 
 ### MQTT
+
 `MQTT` is a lightweight communication protocol which can be used in the IoT scenarios. `Tangle-accelerator`'s support to `MQTT` allows embedded devices to write data on IOTA internet with relative low quality hardware devices. We hope this will speed up DLT into our daily life.
 
 ## Documentation
 
 This page contains basic instructions for setting up tangle-accelerator, You can generate full documentation and API reference via Doxygen. The documentation is under `docs/` after generated:
 
-```
+```bash
 $ doxygen Doxyfile
 ```
 
@@ -90,74 +93,112 @@ Tangle-accelerator is built and launched through Bazel, it also requires Redis t
 
 ## Build from Source
 
-Before running tangle-accelerator, please edit binding address/port of accelerator instance, IRI, and redis server in `accelerator/config.h` unless they are all localhost and/or you don't want to provide external connection. With dependency of [entangled](https://github.com/iotaledger/entangled), IRI address doesn't support https at the moment. Here are some configurations and command you might need to change and use:
+Before running tangle-accelerator, please edit binding address/port of accelerator instance, IOTA full node, and redis server in `accelerator/config.h` unless they are all localhost and/or you don't want to provide external connection. With dependency of [iota.c](https://github.com/iotaledger/iota.c), IOTA full node address doesn't support https at the moment. Here are some configurations and command you might need to change and use:
 
-* `TA_HOST`: binding address of accelerator instance
-* `TA_PORT`: port of accelerator instance
-* `IRI_HOST`: binding address of IRI
-* `IRI_PORT`: port of IRI
-* `quiet`: Turn off logging message
+* `ta_host`: Binding address of accelerator instance.
+* `ta_port`: Port of accelerator instance.
+* `node_host`: Binding address of IOTA full node which includes IRI and Hornet or other community implementation.
+* `node_port`: Port of IOTA full node.
+* `http_threads`: Determine thread pool size to process HTTP connections.
+* `quiet`: Turn off logging message.
 
-```
+```bash
 $ make && bazel run //accelerator
 ```
+
 ### Building Options
+
 Tangle-accelerator supports several different build time options.
 
 * Docker images
 * MQTT connectivity
 * External database
+* Debug Mode
+
+Debug mode enables tangle-accelerator to display extra `debug` logs.
+
+```bash
+$ bazel run --define build_type=debug //accelerator
+```
+
+* Profiling Mode
+
+Profiling mode adds `-pg` flag when compiling tangle-accelerator. This allows tangle-accelerator to write profile information for the analysis program gprof.
+
+```bash
+$ bazel run --define build_type=profile //accelerator
+```
 
 See [docs/build.md](docs/build.md) for more information.
 
 ## Developing
 
 The codebase of this repository follows [Google's C++ guidelines](https://google.github.io/styleguide/cppguide.html):
-- Please run `hooks/autohook.sh install` after initial checkout.
-- Pass `-c dbg` for building with debug symbols.
+
+* Please run `hooks/autohook.sh install` after initial checkout.
+* Pass `-c dbg` for building with debug symbols.
 
 ### Tools required for running git commit hook
-- buildifier
-- clang-format
+
+* buildifier
+* clang-format
+* [shfmt](https://github.com/mvdan/sh)
 
 ### Buildifier
+
 Buildifier can be installed with `bazel` or `go`
 
 #### Install with go
+
 1. change directory to `$GOPATH`
 2. run `$ go get github.com/bazelbuild/buildtools/buildifier`
    The executable file will be located under `$GOPATH/bin`
 3. make a soft link for global usage, run
    `$ sudo ln -s $HOME/go/bin/buildifier /usr/bin/buildifier`
 
 #### Install with bazel
+
 1. clone `bazelbuild/buildtools` repository
    `$ git clone https://github.com/bazelbuild/buildtools.git`
 2. change directory to `buildtools`
 3. build it with bazel command, `$ bazel build //buildifier`
    The executable file will be located under `path/to/buildtools/bazel-bin`
-4. make a soft link or move the executable file under `/usr/bin` 
+4. make a soft link or move the executable file under `/usr/bin`
 
 ### clang-format
+
 clang-format can be installed by command:
-- Debian/Ubuntu based systems: `$ sudo apt-get install clang-format`
-- macOS: `$ brew install clang-format`
 
+* Debian/Ubuntu based systems: `$ sudo apt-get install clang-format`
+* macOS: `$ brew install clang-format`
+
+### shfmt
+
+It requires Go 1.13 or above, and install it with following command.
+
+```shell
+GO111MODULE=on go get mvdan.cc/sh/v3/cmd/shfmt
+```
 
 ## Usage
+
 `Tangle-accelerator` currently supports two categories of APIs
+
 * Direct API: check [wiki page](https://github.com/DLTcollab/tangle-accelerator/wiki) for details.
-* Proxy API to IRI core functionalities
+* Proxy API to IOTA core functionalities
+
+### Full Node Proxy API
 
-### IRI Proxy API
-`tangle-accelerator` allows the use of IRI core APIs. The calling process does not have to be aware of the destination machine running IRI. With the exactly same format of IRI API, `tangle-accelerator` would help users forward the request to IRI and forward the response back to users.
-We support two way to forward Proxy APIs to IRI:
-1. Bypass Proxy APIs directly to IRI.
-2. Process the Proxy APIs, then transmit them to IRI.
+`tangle-accelerator` allows the use of IOTA core APIs. The calling process does not have to be aware of the destination machine running IOTA full node. With the exactly same format of IOTA core APIs, `tangle-accelerator` would help users forward the request to IOTA full node and forward the response back to users.
+We support two way to forward Proxy APIs to IOTA full node:
+
+1. Bypass Proxy APIs directly to IOTA full node.
+2. Process the Proxy APIs, then transmit them to IOTA full node.
 
 The user can choose which way they want with CLI argument `--proxy_passthrough`.
 All the Proxy APIs are supported with the first way.
 However, the second way currently only supports the followings Proxy APIs:
+
 * checkConsistency
 * findTransactions
 * getBalances
@@ -166,5 +207,6 @@ However, the second way currently only supports the followings Proxy APIs:
 * getTrytes
 
 ## Licensing
+
 `Tangle-accelerator` is freely redistributable under the MIT License. Use of this source
 code is governed by a MIT-style license that can be found in the `LICENSE` file.