Merge pull request #171 from scaleoutsystems/release/v0.3.0

release/v0.3.0

Author: dstoyanova

.github/workflows/publish-docs.yaml

@@ -125,19 +125,6 @@ You can lint or check the deployment with the flags —dry-run —debug.
 
 Make sure to assign the chart to the right namespace with —namespace yournamespace (when deploying to the default namespace this can be omitted.)
 
-### 5. Setup a user
-
-You will need to create a user to login into Studio. Click the login button in the lower left corner, and click register. By default, Keycloak is configured not to require email verification, but this can be changed by logging into the Keycloak admin console and updating the STACKn realm login settings.
-
-To access the admin page of Studio, you will need to create a Django user with admin rights. First find the pod name to the Studio deployment:
-```bash
-$ kubectl get pods -n yournamespace
-```
-and get the pod id that correspond to the studio pod running. Replace `pod-Id` in the command below.
-```bash
-$ kubectl exec -it pod-Id python manage.py createsuperuser
-```
-
 ### Additional - Upgrading STACKn
 
 Similar to how you install a chart you may also upgrade a chart.
@@ -175,4 +162,4 @@ STACKn is used in various places, examples include [SciLifeLab Data Center](http
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 ## License
-> See [LICENSE](LICENCE.md) for details.
+> See [LICENSE](LICENSE) for details.

README.md

@@ -142,21 +142,29 @@ def get_token(client_id='studio-api', realm='STACKn', secure=True):
         else:
             print('Failed to authenticate with token, please login again.')
             print(res.text)
-            access_token = login()
+            access_token = login(deployment=stackn_config['active'], keycloak_host=token_config['keycloak_url'], studio_host=token_config['studio_url'], secure=secure)
 
     return access_token, token_config
 
 
 
-def login(client_id='studio-api', realm='STACKn', deployment=[], keycloak_host=[], studio_host=[], secure=True):
+def login(client_id='studio-api', realm='STACKn', deployment=[], keycloak_host=[], studio_host=[], username=[], secure=True):
     """ Login to Studio services. """
     if not deployment:
         deployment = input('Name: ')
-    if not keycloak_host:
-        keycloak_host = input('Keycloak host: ')
     if not studio_host:
         studio_host = input('Studio host: ')
-    username = input('Username: ')
+
+    url = "{}/api/settings".format(studio_host)
+    r = requests.get(url)
+    if (r.status_code >= 200 or r.status_code <= 299):
+        studio_settings = json.loads(r.content)["data"]
+        keycloak_host = next(item for item in studio_settings if item["name"] == "keycloak_host")["value"]
+
+    if not keycloak_host:
+        keycloak_host = input('Keycloak host: ')
+    if not username:
+        username = input('Username: ')
     password = getpass()
     access_token, refresh_token, public_key = keycloak_user_auth(username, password, keycloak_host, secure=secure)
     # dirname = base64.urlsafe_b64encode(host.encode("utf-8")).decode("utf-8")

cli/scaleout/auth.py

@@ -4,7 +4,7 @@
 from .create_cmd import create_cmd
 from .get_cmd import get_cmd
 from .delete_cmd import delete_cmd
-from .stackn_cmd import setup_cmd, status_cmd, predict_cmd
+from .stackn_cmd import setup_cmd, status_cmd, predict_cmd, train_cmd, test_cmd
 from .set_cmd import set_cmd
 from .update_cmd import update_cmd
 from .init_cmd import init_cmd

cli/scaleout/cli/__init__.py

@@ -63,11 +63,26 @@ def create_project_cmd(ctx, name, description='', repository=''):
 @create_cmd.command('lab')
 @click.option('-f', '--flavor', required=True)
 @click.option('-e', '--environment', required=True)
+@click.option('-v', '--volumes', required=False, default=[])
 @click.pass_context
-def create_session(ctx, flavor, environment):
+def create_session(ctx, flavor, environment, volumes):
     client = ctx.obj['CLIENT']
-    client.create_session(flavor_slug=flavor, environment_slug=environment)
+    client.create_session(flavor_slug=flavor, environment_slug=environment, volumes=volumes)
 
+@create_cmd.command('volume')
+@click.option('-s', '--size', required=True)
+@click.option('-n', '--name', required=True)
+@click.pass_context
+def create_volume(ctx, size, name):
+    client = ctx.obj['CLIENT']
+    client.create_volume(name=name, size=size)
+
+@create_cmd.command('job')
+@click.option('-c', '--config', required=True)
+@click.pass_context
+def create_job(ctx, config):
+    client = ctx.obj['CLIENT']
+    client.create_job(config)
 
 # Create dataset
 
@@ -85,4 +100,4 @@ def create_dataset(ctx, name, directory=[], filenames=[], release_type='minor',
                           filenames,
                           directory,
                           description=description,
-                          bucket=bucket)
+                          bucket=bucket)

cli/scaleout/cli/create_cmd.py

@@ -44,6 +44,14 @@ def delete_dataset_cmd(ctx, name, version=None):
     client = ctx.obj['CLIENT']
     client.delete_dataset(name, version)
 
+@delete_cmd.command('volume')
+@click.option('-n', '--name', required=True)
+@click.pass_context
+def delete_volume_cmd(ctx, name):
+    """ Delete a volume """
+    client = ctx.obj['CLIENT']
+    client.delete_volume(name)
+
 # @delete_cmd.command('deployments')
 # @click.pass_context
 # def delete_deployment_cmd(ctx):

cli/scaleout/cli/default-project.tar.gz

@@ -1,8 +1,8 @@
+import json
 import click
 from .main import main
 import requests
-from scaleout.studioclient import StudioClient 
-from .helpers import create_table
+from .helpers import create_table, PrettyTable
 
 @click.option('--daemon',
               is_flag=True,
@@ -17,6 +17,32 @@ def get_cmd(ctx, daemon):
     if daemon:
         print('{} NYI should run as daemon...'.format(__file__))
 
+@get_cmd.command('settings')
+@click.pass_context
+def get_settings_cmd(ctx):
+    """
+    List STACKn settings needed to set up the CLI client.
+    """
+    studio_host = input("Studio host: ")
+    url = "{}/api/settings".format(studio_host)
+    try:
+        r = requests.get(url)
+        studio_settings = json.loads(r.content)["data"]
+
+        names = ['Setting', 'Value']
+        keys = ['name', 'value']
+        x = PrettyTable()
+        x.field_names = names
+        for item in studio_settings:
+            row = [item[k] for k in keys]
+            x.add_row(row)
+        print(x)
+    except Exception as e:
+        print("Couldn't get studio settings.")
+        print("Returned status code: {}".format(r.status_code))
+        print("Reason: {}".format(r.reason))
+        print("Error: {}".format(e))
+
 @get_cmd.command('models')
 @click.pass_context
 def get_models_cmd(ctx):
@@ -44,6 +70,7 @@ def get_deploymentdefinitions_cmd(ctx):
 @get_cmd.command('projects')
 @click.pass_context
 def get_projects_cmd(ctx):
+    """ List all projects. """
     names = ["Name","Created", "Last updated"]
     keys = ["name", "created_at", "updated_at"]
     create_table(ctx, "projects", names, keys)
@@ -56,6 +83,22 @@ def lab_list_all_cmd(ctx):
     keys = ["name", "flavor_slug", "environment_slug", "status", "created_at"]
     create_table(ctx, "labs", names, keys)
 
+@get_cmd.command('volumes')
+@click.pass_context
+def get_volumes_cmd(ctx):
+    """ List all volumes """
+    names = ["Name","Size", "Created by","Created"]
+    keys = ['name', 'size', 'created_by', 'created_on']
+    create_table(ctx, 'volumes', names, keys)
+
+@get_cmd.command('jobs')
+@click.pass_context
+def get_jobs_cmd(ctx):
+    """ List all jobs """
+    names = ["User","command", "Environment","Schedule"]
+    keys = ['username', 'command', 'environment', 'schedule']
+    create_table(ctx, 'jobs', names, keys)
+
 @get_cmd.command('members')
 @click.pass_context
 def members_list_cmd(ctx):

cli/scaleout/cli/delete_cmd.py

@@ -1,5 +1,7 @@
 import sys
 from prettytable import PrettyTable
+import click
+import uuid
 
 
 def prompt(question, default="yes"):
@@ -45,4 +47,47 @@ def _print_table(resource, names, keys):
 def create_table(ctx, resource, names, keys):
     client = ctx.obj['CLIENT']
     objects = client.create_list(resource)
-    _print_table(objects, names, keys)
+    _print_table(objects, names, keys)
+
+def search_for_model(ctx, resource, name):
+    client = ctx.obj['CLIENT']
+    objects = client.create_list(resource)
+    model_exists = False
+    for item in objects:
+        if item['name'] == name:
+            model_exists = True
+    return model_exists
+ 
+def new_id(run_id):
+    new_id = input("A log object with ID = {} already exists in 'src/models/tracking' directory. \n".format(run_id) \
+                 + "Please provide a unique ID for the current run or press enter to use a randomly generated ID: ")
+    if new_id:
+        confirmed = False
+        question = "Do you want to assign this training run with the ID '{}'?".format(new_id)
+        while not confirmed:
+            confirmed = prompt(question)
+            if confirmed:
+                return new_id
+            else:
+                new_id = input("Assign a new unique ID or press enter to assign a random ID: ")
+                print(new_id)
+                if not new_id:
+                    break
+    new_id = str(uuid.uuid1().hex)
+    return new_id
+
+class Determinant(click.Option):
+    def __init__(self, *args, **kwargs):
+        self.determinant = kwargs.pop('determinant')
+        assert self.determinant, "'determinant' parameter required"
+        super(Determinant, self).__init__(*args, **kwargs)
+
+    def handle_parse_result(self, ctx, opts, args):
+        unallowed_present = self.name in opts
+        determinant_present = self.determinant in opts
+        if determinant_present:
+            if unallowed_present:
+                raise click.UsageError("Illegal usage: Cannot pass a value for '{}' together with '{}' when running 'stackn train'".format(self.name, self.determinant))
+            else:
+                self.prompt = None
+        return super(Determinant, self).handle_parse_result(ctx, opts, args)

cli/scaleout/cli/get_cmd.py

@@ -2,7 +2,13 @@
 from .main import main
 import requests
 from scaleout.auth import login, get_stackn_config, get_remote_config, get_token
-from .helpers import create_table
+from .helpers import create_table, search_for_model, new_id, Determinant
+import os
+import random
+import string
+import json
+import uuid
+from scaleout.details import get_run_details 
 
 # @click.option('--daemon',
 #               is_flag=True,
@@ -57,3 +63,45 @@ def predict_cmd(ctx, model, version, inp):
     # res = requests.post(url,
     #                     headers={"Authorization": "Token "+token},
     #                     json = inp)
+
+
+# ------------------- Question ---------------------
+# Is it a good idea to make it possible to pass --log-off as an argument if the user does not want to log the run to Studio?
+# In that case, the model name and code version is not possible to pass to the stackn train command and train.py will run without logging. 
+# Not sure if this is a good idea
+# --------------------------------------------------
+@main.command('train')
+@click.option('--log-off', flag_value='log-off', default=False)
+@click.option('-m', '--model', prompt=True, cls=Determinant, determinant='log_off')
+@click.option('-i', '--run-id', required=False, default=str(uuid.uuid1().hex))
+@click.option('-f', '--training-file', required=False, default="src/models/train.py")
+@click.option('-v', '--version', prompt=True, cls=Determinant, determinant='log_off')
+@click.pass_context
+def train_cmd(ctx, log_off, model, run_id, training_file, version):
+    """ Train a model and log metadata """
+    
+    if os.path.isfile('src/models/tracking/metadata/{}.pkl'.format(run_id)): # Only checks locally. Should we check if there exists a log on Studio with the same ID as well?
+        run_id = new_id(run_id)
+    print("Preparing to start training session with '{}' as unique ID.".format(run_id))
+    if os.path.isfile(training_file):
+        if log_off:
+            import subprocess
+            subprocess.run(['python', training_file, run_id])
+        else:
+            model_exists = search_for_model(ctx, "models", model) 
+            if model_exists:
+                client = ctx.obj['CLIENT']
+                client.train(model, run_id, training_file, version)
+            else: 
+                print("The model '{}' does not exist in the active project and cannot be trained.".format(model))
+    else:
+        current_dir = os.getcwd()
+        print("Could not start a training session. Check that you have initialized a model "\
+            + "in '{}' and that the file '{}' exists.".format(current_dir, training_file))    
+
+
+
+@main.command('test')
+@click.pass_context
+def test_cmd(ctx):
+    get_run_details('12')