Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .dir-locals.el
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
;;; For more information see (info "(emacs) Directory Variables")

((nil .
((cider-clojure-cli-global-options . "-A:cli:dev:test:onyx:onyx-dev"))))
((cider-clojure-cli-global-options . "-A:cli:dev:test"))))
32 changes: 32 additions & 0 deletions .github/workflows/deps.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Deps

on:
push:
branches:
- '*'
jobs:
deps:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout project
uses: actions/checkout@v4

- name: Setup CI Environment
uses: yetanalytics/action-setup-env@v2

- name: Cache Deps
uses: actions/cache@v4
with:
path: |
~/.m2
~/.gitlibs
key: ${{ runner.os }}-deps-${{ hashFiles('deps.edn') }}
restore-keys: |
${{ runner.os }}-deps-

- name: Make a POM
run: make clean pom.xml

- name: Submit Dependency Snapshot
uses: advanced-security/maven-dependency-submission-action@v4
16 changes: 0 additions & 16 deletions .github/workflows/nvd.yml

This file was deleted.

6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ test-cli-comprehensive:
clojure -A:cli:run-cli validate-input -i dev-resources/input/simple.json -v dev-resources/input/simple.json

test-cli-output:
clojure -A:cli:run-cli generate -i dev-resources/input/simple.json
clojure -A:cli:run-cli generate -i dev-resources/input/simple.json

test-bundle-output: bundle
cd target/bundle; bin/run.sh generate -i ../../dev-resources/input/simple.json
Expand All @@ -64,3 +64,7 @@ ci: test-unit test-unit-onyx test-cli validate-template

server:
clojure -A:server:run-server

# For dependabot
pom.xml:
clojure -A:cli:server:test:dev:onyx -Spom
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,9 @@ The `generate-post` subcommand is used to generate statements from an input and
| `-E, --endpoint URI` | The xAPI endpoint of an LRS to POST to, ex: `https://lrs.example.org/xapi` | N/A
| `-U, --username URI` | The Basic Auth username for the LRS. | N/A
| `-P, --password URI` | The Basic Auth password for the LRS. | N/A
| `--token` | Bearer token to use for LRS auth. | N/A
| `--cookie` | Cookie to use for LRS auth. | N/A
| `--credential-id` | Credential ID to use for LRS cookie auth. | N/A
| `-B, --batch-size SIZE` | The batch size, i.e. how many statements to send at a time, for POSTing. | `25`
| `-C, --concurrency CONC` | The max concurrency of the LRS POST pipeline. | `4`
| `-L, --post-limit LIMIT` | The total number of statements that will be sent to the LRS before termination. Overrides sim params. Set to -1 for no limit. | `999`
Expand All @@ -241,7 +244,7 @@ bin/run.sh validate-input \
-p dev-resources/profile/cmi5/fixed.json \
-a dev-resources/personae/simple.json \
-m dev-resources/models/simple.json \
-o dev-resources/parameters/simple.json \
-o dev-resources/parameters/simple.json \
-v dev-resources/input/simple.json
```

Expand Down
32 changes: 28 additions & 4 deletions src/cli/com/yetanalytics/datasim/cli/generate.clj
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@
(def password-desc
"The Basic Auth password for the LRS.")

(def token-desc
"An optional bearer token to use in the Authorization header (overrides username and password if present).")

(def cookie-desc
"An optional cookie string to send in the Cookie header.")

(def credential-id-desc
"An optional credentialID parameter to send to the LRS.")

(def batch-size-desc
"The batch size, i.e. how many statements to send at a time, for POSTing.")

Expand All @@ -76,6 +85,15 @@
["-P" "--password URI" "LRS password"
:id :password
:desc password-desc]
[nil "--token TOKEN" "Bearer Token"
:id :token
:desc token-desc]
[nil "--cookie COOKIE" "Authentication Cookie"
:id :cookie
:desc cookie-desc]
[nil "--credential-id UUID" "Yet LRS Credential ID for use with cookie auth."
:id :credential-id
:desc credential-id-desc]
["-B" "--batch-size SIZE" "LRS POST batch size"
:id :batch-size
:default 25
Expand Down Expand Up @@ -154,17 +172,23 @@
(let [{:keys [endpoint
username
password
token
cookie
credential-id
batch-size
concurrency
post-limit
select-agents
async]}
options
post-options
{:endpoint endpoint
:batch-size batch-size
:username username
:password password}]
{:endpoint endpoint
:batch-size batch-size
:username username
:password password
:token token
:cookie cookie
:credential-id credential-id}]
(if async
(post-async! input post-options post-limit select-agents concurrency)
(post-sync! input post-options post-limit select-agents))))
Expand Down
2 changes: 1 addition & 1 deletion src/cli/com/yetanalytics/datasim/cli/input.clj
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@
(defn validate-input*
"Perform validation on `input` and fail w/ early termination if
it is not valid.

When this is called, we should have valid individual inputs. However, there
may be cross-validation that needs to happen, so we compose the
comprehensive spec from the options and check that."
Expand Down
45 changes: 32 additions & 13 deletions src/main/com/yetanalytics/datasim/client.clj
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
(format "%s/statements" endpoint))

(defn- post-options [http-options batch]
(merge default-http-options
http-options
{:body (json/encode batch)
:as :stream}))
(merge-with merge
default-http-options
http-options
{:body (json/encode batch)
:as :stream}))

;; `http/post` cannot be resolved since it's defined using `http/defreq`
#_{:clj-kondo/ignore [:unresolved-var]}
Expand All @@ -41,21 +42,39 @@
(post-options http-options batch)
callback-fn)))

(defn- auth-options
[{:keys [username
password
token
cookie
credential-id]}]
(let [opts (cond
(and username password)
{:basic-auth [username password]}
token
{:headers {"Authorization" (format "Bearer %s" token)}}
cookie
(cond-> {:headers {"Cookie" cookie}}
credential-id
(assoc :query-params {:credentialID credential-id}))
:else {})]
(println opts)
opts))

(defn post-statements
"Given LRS options and a `statement-seq`, send them to an LRS in synchronous
batches. If `print-ids?` is `true`, returned statement IDs will be printed
to stdout. `username` and `password` in the options map are the Basic Auth
credentials of the LRS."
[{:keys [endpoint
batch-size
username
password]
batch-size]
:as options
:or {batch-size 25}}
statement-seq
& {:keys [print-ids?]
:or {print-ids? true}}]
;; TODO: Exponential backoff, etc
(let [http-options {:basic-auth [username password]}]
(let [http-options (auth-options options)]
(loop [batches (partition-all batch-size statement-seq)
success 0
fail []]
Expand All @@ -82,14 +101,14 @@
(defn post-statements-async
"Given LRS options and a channel with statements, send them to an LRS in
asynchronous batches. `username` and `password` in the options map are the
Basic Auth credentials of the LRS.
Basic Auth credentials of the LRS. Other auth methods are supported via
`token` and `cookie`.

Returns a channel that will reciveve `[:success <list of statement ids>]`
for each batch or `[:fail <failing request>]`. Will stop sending on failure."
[{:keys [endpoint
batch-size
username
password]
batch-size]
:as options
:or {batch-size 25}}
statement-chan
& {:keys [concurrency
Expand All @@ -98,7 +117,7 @@
:or {concurrency 4
buffer-in 100 ; 10x default batch size
buffer-out 100}}]
(let [http-opts {:basic-auth [username password]}
(let [http-opts (auth-options options)
run? (atom true)
in-chan (a/chan buffer-in (partition-all batch-size))
out-chan (a/chan buffer-out) ; is this.. backpressure?
Expand Down
Loading