diff --git a/.dir-locals.el b/.dir-locals.el index 238cc457..abb1dba5 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -2,4 +2,4 @@ ;;; For more information see (info "(emacs) Directory Variables") ((nil . - ((cider-clojure-cli-global-options . "-A:cli:dev:test:onyx:onyx-dev")))) + ((cider-clojure-cli-global-options . "-A:cli:dev:test")))) diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml new file mode 100644 index 00000000..79b524d8 --- /dev/null +++ b/.github/workflows/deps.yml @@ -0,0 +1,32 @@ +name: Deps + +on: + push: + branches: + - '*' +jobs: + deps: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout project + uses: actions/checkout@v4 + + - name: Setup CI Environment + uses: yetanalytics/action-setup-env@v2 + + - name: Cache Deps + uses: actions/cache@v4 + with: + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }} + restore-keys: | + ${{ runner.os }}-deps- + + - name: Make a POM + run: make clean pom.xml + + - name: Submit Dependency Snapshot + uses: advanced-security/maven-dependency-submission-action@v4 diff --git a/.github/workflows/nvd.yml b/.github/workflows/nvd.yml deleted file mode 100644 index 0146641a..00000000 --- a/.github/workflows/nvd.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Periodic NVD Scan - -on: - schedule: - - cron: '0 8 * * 1-5' # Every weekday at 8:00 AM - -jobs: - nvd-scan: - uses: yetanalytics/workflow-nvd/.github/workflows/nvd-scan.yml@v1 - with: - nvd-clojure-version: '3.2.0' - classpath-command: 'clojure -Spath -A:cli:server' - nvd-config-filename: '.nvd/config.json' - notify-slack: true - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/Makefile b/Makefile index 66a7053a..8d8dc6be 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ test-cli-comprehensive: clojure -A:cli:run-cli validate-input -i dev-resources/input/simple.json -v dev-resources/input/simple.json test-cli-output: - clojure -A:cli:run-cli generate -i dev-resources/input/simple.json + clojure -A:cli:run-cli generate -i dev-resources/input/simple.json test-bundle-output: bundle cd target/bundle; bin/run.sh generate -i ../../dev-resources/input/simple.json @@ -64,3 +64,7 @@ ci: test-unit test-unit-onyx test-cli validate-template server: clojure -A:server:run-server + +# For dependabot +pom.xml: + clojure -A:cli:server:test:dev:onyx -Spom diff --git a/README.md b/README.md index 65bf24bd..9e99302f 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,9 @@ The `generate-post` subcommand is used to generate statements from an input and | `-E, --endpoint URI` | The xAPI endpoint of an LRS to POST to, ex: `https://lrs.example.org/xapi` | N/A | `-U, --username URI` | The Basic Auth username for the LRS. | N/A | `-P, --password URI` | The Basic Auth password for the LRS. | N/A +| `--token` | Bearer token to use for LRS auth. | N/A +| `--cookie` | Cookie to use for LRS auth. | N/A +| `--credential-id` | Credential ID to use for LRS cookie auth. | N/A | `-B, --batch-size SIZE` | The batch size, i.e. how many statements to send at a time, for POSTing. | `25` | `-C, --concurrency CONC` | The max concurrency of the LRS POST pipeline. | `4` | `-L, --post-limit LIMIT` | The total number of statements that will be sent to the LRS before termination. Overrides sim params. Set to -1 for no limit. | `999` @@ -241,7 +244,7 @@ bin/run.sh validate-input \ -p dev-resources/profile/cmi5/fixed.json \ -a dev-resources/personae/simple.json \ -m dev-resources/models/simple.json \ - -o dev-resources/parameters/simple.json \ + -o dev-resources/parameters/simple.json \ -v dev-resources/input/simple.json ``` diff --git a/src/cli/com/yetanalytics/datasim/cli/generate.clj b/src/cli/com/yetanalytics/datasim/cli/generate.clj index d621db4e..60983cf8 100644 --- a/src/cli/com/yetanalytics/datasim/cli/generate.clj +++ b/src/cli/com/yetanalytics/datasim/cli/generate.clj @@ -56,6 +56,15 @@ (def password-desc "The Basic Auth password for the LRS.") +(def token-desc + "An optional bearer token to use in the Authorization header (overrides username and password if present).") + +(def cookie-desc + "An optional cookie string to send in the Cookie header.") + +(def credential-id-desc + "An optional credentialID parameter to send to the LRS.") + (def batch-size-desc "The batch size, i.e. how many statements to send at a time, for POSTing.") @@ -76,6 +85,15 @@ ["-P" "--password URI" "LRS password" :id :password :desc password-desc] + [nil "--token TOKEN" "Bearer Token" + :id :token + :desc token-desc] + [nil "--cookie COOKIE" "Authentication Cookie" + :id :cookie + :desc cookie-desc] + [nil "--credential-id UUID" "Yet LRS Credential ID for use with cookie auth." + :id :credential-id + :desc credential-id-desc] ["-B" "--batch-size SIZE" "LRS POST batch size" :id :batch-size :default 25 @@ -154,6 +172,9 @@ (let [{:keys [endpoint username password + token + cookie + credential-id batch-size concurrency post-limit @@ -161,10 +182,13 @@ async]} options post-options - {:endpoint endpoint - :batch-size batch-size - :username username - :password password}] + {:endpoint endpoint + :batch-size batch-size + :username username + :password password + :token token + :cookie cookie + :credential-id credential-id}] (if async (post-async! input post-options post-limit select-agents concurrency) (post-sync! input post-options post-limit select-agents)))) diff --git a/src/cli/com/yetanalytics/datasim/cli/input.clj b/src/cli/com/yetanalytics/datasim/cli/input.clj index 00f9ed98..5842703b 100644 --- a/src/cli/com/yetanalytics/datasim/cli/input.clj +++ b/src/cli/com/yetanalytics/datasim/cli/input.clj @@ -120,7 +120,7 @@ (defn validate-input* "Perform validation on `input` and fail w/ early termination if it is not valid. - + When this is called, we should have valid individual inputs. However, there may be cross-validation that needs to happen, so we compose the comprehensive spec from the options and check that." diff --git a/src/main/com/yetanalytics/datasim/client.clj b/src/main/com/yetanalytics/datasim/client.clj index acb05368..c721e9d5 100644 --- a/src/main/com/yetanalytics/datasim/client.clj +++ b/src/main/com/yetanalytics/datasim/client.clj @@ -25,10 +25,11 @@ (format "%s/statements" endpoint)) (defn- post-options [http-options batch] - (merge default-http-options - http-options - {:body (json/encode batch) - :as :stream})) + (merge-with merge + default-http-options + http-options + {:body (json/encode batch) + :as :stream})) ;; `http/post` cannot be resolved since it's defined using `http/defreq` #_{:clj-kondo/ignore [:unresolved-var]} @@ -41,21 +42,39 @@ (post-options http-options batch) callback-fn))) +(defn- auth-options + [{:keys [username + password + token + cookie + credential-id]}] + (let [opts (cond + (and username password) + {:basic-auth [username password]} + token + {:headers {"Authorization" (format "Bearer %s" token)}} + cookie + (cond-> {:headers {"Cookie" cookie}} + credential-id + (assoc :query-params {:credentialID credential-id})) + :else {})] + (println opts) + opts)) + (defn post-statements "Given LRS options and a `statement-seq`, send them to an LRS in synchronous batches. If `print-ids?` is `true`, returned statement IDs will be printed to stdout. `username` and `password` in the options map are the Basic Auth credentials of the LRS." [{:keys [endpoint - batch-size - username - password] + batch-size] + :as options :or {batch-size 25}} statement-seq & {:keys [print-ids?] :or {print-ids? true}}] ;; TODO: Exponential backoff, etc - (let [http-options {:basic-auth [username password]}] + (let [http-options (auth-options options)] (loop [batches (partition-all batch-size statement-seq) success 0 fail []] @@ -82,14 +101,14 @@ (defn post-statements-async "Given LRS options and a channel with statements, send them to an LRS in asynchronous batches. `username` and `password` in the options map are the - Basic Auth credentials of the LRS. + Basic Auth credentials of the LRS. Other auth methods are supported via + `token` and `cookie`. Returns a channel that will reciveve `[:success ]` for each batch or `[:fail ]`. Will stop sending on failure." [{:keys [endpoint - batch-size - username - password] + batch-size] + :as options :or {batch-size 25}} statement-chan & {:keys [concurrency @@ -98,7 +117,7 @@ :or {concurrency 4 buffer-in 100 ; 10x default batch size buffer-out 100}}] - (let [http-opts {:basic-auth [username password]} + (let [http-opts (auth-options options) run? (atom true) in-chan (a/chan buffer-in (partition-all batch-size)) out-chan (a/chan buffer-out) ; is this.. backpressure?