dsci-runner.git | common/ | sparrowfile

---

#!raku

use Sparky::JobApi;

use JSON::Fast;

use YAMLish;

class Pipeline

does Sparky::JobApi::Role

{

  method !version {
      return "0.1.0"
  }

  method !validate-cond-expression ($exp,$debug = False) {
    my @b; # bogus lines
    for $exp.split(/\s [ "and" || "or" || "&&" || "||" ]  \s/)
    .map({
      .subst("(","",:g)
      .subst(")","",:g)
      .subst(/^^ \s+/,"")
      .subst(/\s+ $$/,"")
      .subst('$$',"rx_end_of_line",:g)
      }) -> $i {
      say "parse [$i]" if $debug;  
      if  $i !~~ /
          ^^ 
            ".<" 
              \w+ 
            ">" 
          \s+ 
          [ 
            ">" || ">=" || "<" || "<=" || "==" || "!=" || "eq" || "ne" ||  "~~" || "!~~" 
          ]
          \s+
          [
            \" <[ \/ \\ \d \. \: \w \d ]> + \" || 
            \d+ || 
            "m:" <[i m g r s \:]> + \s* \/ <-[ \{ \{ \$  ]> +  \/ ||
            \/ <-[ \{ \{ \$  ]> + \/
          ] 
          $$
        / {
        push @b, $i
      }
    }
    if @b.elems {
      say "validate-cond-expression: does not look good";
      say "validated expression: $exp";
      say "===";
      for @b -> $i {
        say "bogus chunk found: $i"
      }
      say "===";
    }
    return @b.elems == 0;
  }

  method stage-main {

    directory "scm";

    my $cr = tags()<cr> || "docker"; # container runtime: docker|podman

    say "scm: ",tags()<scm>;
    say "sha: ",tags()<sha>;
    say "message: ",tags()<message>.chomp;
    say "container runtime: ", $cr;
    say "runner version: {self!version()}";
    say "=======";

    bash "git config --global advice.detachedHead false";

    my $scm = tags()<scm>;

    git-scm tags()<scm>, %(
      :to<scm>,
      :branch(tags()<sha>),
      :skip-pull,
    );

    my $jobs = load-yaml("scm/.dsci/jobs.yaml".IO.slurp);

    my $status = True;

    #say $jobs.raku;

    my $cont = True;

    my $localhost-mode = $jobs<global> && $jobs<global><localhost>;

    say "pipeline mode: {$localhost-mode ?? 'localhost' !! 'docker'}";

    if $localhost-mode and ! tags()<DsciAllowLocalhostModeRepos> {
      die "localhost is forbidden, DsciAllowLocalhostModeRepos is not set";
      #$status = False;
      #$cont = False; 
    } elsif $localhost-mode and tags()<DsciAllowLocalhostModeRepos> {
      my @repos = tags()<DsciAllowLocalhostModeRepos>.split(":");
      $status = False;
      for @repos -> $i {
        if tags()<repo_full_name> eq $i {
          $status = True;
          last();
        }
      }
      unless $status {
        die "localhost is forbidden, {tags()<repo_full_name>} should be listed in DsciAllowLocalhostModeRepos";
        #$cont = False;
      }
    }

    if $cont && $jobs<global> && $jobs<global><skip> {
      if self!validate-cond-expression($jobs<global><skip>,$jobs<global><debug>||False) {
        use MONKEY-SEE-NO-EVAL;
        $cont = False if EVAL("my \$a = \{given (tags()) \{ {$jobs<global><skip>} \}\}; \$a()");
      } else {
        warn "skip: job condition validation failed";
        $status = False;
        $cont = False; 
      }
    }

    if $cont && $jobs<global> && $jobs<global><only> {
      if self!validate-cond-expression($jobs<global><only>,$jobs<global><debug>||False) {
        use MONKEY-SEE-NO-EVAL;
        $cont = False unless EVAL("my \$a = \{given (tags()) \{ {$jobs<global><only>} \}\}; \$a()");
      } else {
        warn "only: job condition validation failed";
        $status = False;
        $cont = False; 
      }
    }

    if $cont {

      my %_dsci;
    
      if $localhost-mode {

        unless tags()<DsciAgentSkipBootstrap> {
          my $j = Sparky::JobApi.new(
              :project("job.bootstrap"),
          );

          $j.queue({
              :description("dsci bootstrap"),
              :tags(%(stage => "bootstrap", SPARKY_HOST => "127.0.0.1")),
              :sparrowdo(%(
                :host<127.0.0.1>,
                :ssh_user(%*ENV<HOST_SSH_USER>),
                :bootstrap,
                :no_sudo,
              )),
          });

          my $st = self.wait-job($j);

          if ! $st<OK> {
            $status = False;
            my $eff-status = $st<TIMEOUT> ?? "TIMEOUT" !! "FAIL";
            say "dsci bootstrap ... [$eff-status]";
            say "...";
            say $j.report();
          } else {
            say "dsci bootstrap ... [OK]";
            say "...";
            say $j.report();
          }

        }

      } else {

        if $cr eq "docker" {
          bash "sudo chmod 666 /var/run/docker.sock";
        }

        bash "$cr stop -t 1 dsci-agent || :";

        my @env;

        if "{%*ENV<HOME>}/.secrets/{tags()<repo_full_name>}".IO ~~ :d {
          for dir("{%*ENV<HOME>}/.secrets/{tags()<repo_full_name>}") -> $file {
            #say $file;
            next unless $file ~~ :f;
            push @env, "-e {$file.basename}=\"{$file.slurp.chomp}\"";
            say "push secret {$file.basename} to job container ...";
          }
        }

        say "run dsci agent container from image: {tags()<DsciAgentImage>||'alpine:latest'}";

        say "$cr run --network host  -dit --rm --name dsci-agent --entrypoint /bin/sh {tags()<DsciAgentImage> || 'alpine:latest'}";

        bash "$cr run --network host  -dit --rm --name dsci-agent --entrypoint /bin/sh {@env.join(' ')} {tags()<DsciAgentImage> || 'alpine:latest'}";

        unless tags()<DsciAgentSkipBootstrap> {
          my $j = Sparky::JobApi.new(
              :project("job.bootstrap"),
          );

          $j.queue({
              :description("dsci agent bootstrap"),
              :tags(%(stage => "bootstrap", SPARKY_HOST => "127.0.0.1")),
              :sparrowdo(%(
                :docker<dsci-agent>,
                :bootstrap,
                :no_sudo,
              )),
          });
    

          my $st = self.wait-job($j);

          if ! $st<OK> {
            $status = False;
            my $eff-status = $st<TIMEOUT> ?? "TIMEOUT" !! "FAIL";
            say "dsci agent bootstrap ... [$eff-status]";
            say "...";
            say $j.report();
          } else {
            say "dsci agent bootstrap ... [OK]";
            say "...";
            say $j.report();
          }

        }

      }

      if $status {

        for $jobs<jobs>.flat -> $job {

          if $job<only> {
            if self!validate-cond-expression($job<only>,$job<debug>||False) {
              use MONKEY-SEE-NO-EVAL;
              next unless EVAL("my \$a = \{given (tags()) \{ {$job<only>} \}\}; \$a()");
            } else {
              warn "only: job condition validation failed";
              $status = False;
              last; 
            }
          }

          if $job<skip> {
            if self!validate-cond-expression($job<skip>,$job<debug>||False) {
            use MONKEY-SEE-NO-EVAL;
              next if EVAL("my \$a = \{given (tags()) \{ {$job<skip>} \}\}; \$a()");
            } else {
              warn "skip: job condition validation failed";
              $status = False;
              last; 
            }
          }

          my $j = Sparky::JobApi.new(
            :project("job.run"),
          );

          my %tags = (
            job_id  => $job<id>,
            path    => $job<path>,
            stage   => "job-run",
            scm     => tags()<scm>,
            sha     => tags()<sha>,
            SPARKY_HOST => "127.0.0.1",
            ref     => tags()<ref>,
            repo_full_name  => tags()<repo_full_name>,
            message => tags()<message>,
            dispatcher_job_id => tags()<SPARKY_JOB_ID>,
          );

          if $job<plugin> {
            %tags<plugin> = $job<plugin>
          } elsif $job<path> {
            %tags<path> = $job<path>
          } else {
            $status = False;
            say "neither path nor path is set";
            last;
          }

          if $job<debug> {
            %tags<dump-task-config> = True;
            %tags<dump-task-code> = True;
          }

          # copy dsci internal cache
          # to job input parameters
          # will be available
          # via config()<dsci>
          # object

          if $job<params> {
            $j.put-stash(%(
              params => $job<params>,
              dsci => %_dsci,
            ));
          } else {
            $j.put-stash(%(
              dsci => %_dsci,
              params => Hash.new,
            ));
          }

          my %sparrowdo = %( :no_sudo );
          if $localhost-mode {
            $j.queue({
              :description($job<id>),
              :tags(%tags),
              sparrowdo => %(
                :no_sudo,
                :host<127.0.0.1>,
                :ssh_user("{%*ENV<HOST_SSH_USER>}"),
              ),
            });
          } else {
            $j.queue({
              :description($job<id>),
              :tags(%tags),
              sparrowdo => %( 
                :no_sudo, 
                :docker<dsci-agent>,
              ),
            });
          }

          my $st = self.wait-job($j);

          unless $st<OK> {
            $status = False;
            my $eff-status = $st<TIMEOUT> ?? "TIMEOUT" !! "FAIL";
            say "job {$job<id>} ... [$eff-status]";
            say "...";
            say $j.report();
            last;
          }

          my $job-data = $j.get-stash();

          # copy job state to dsci internal cache
          %_dsci{$job<id>} = $job-data;

          # copy job artifacts (if any) to dsci internal cache
          if $job-data<artifacts> {
            %_dsci<dsci_artifacts> = $job-data<artifacts>
          }

          say "job {$job<id>} ... [OK]";
          say "...";
          say $j.report();

        }
      }

      if $status == False {
        die "some child jobs failed"
      }
    }
  }

  method stage-bootstrap {

    package-install "git";

    #bash "zef install --/test https://github.com/melezhik/Sparrow6.git";

    package-install "python3";

    #bash "zef install --/test --force-install https://github.com/melezhik/sparky-job-api.git", %(
    #  description => "update sparky-job-api",
    #);

    say "dsci agent is ready!!!";

  }

  method stage-job-run {

    #say tags().raku;

    my $me = Sparky::JobApi.new( :mine );

    say "run job: {tags()<job_id>}";

    say "===";

    #say $me.get-stash().raku;

    my $artifacts = $me.get-stash()<dsci><dsci_artifacts>;

    my $job-id = tags()<dispatcher_job_id>;
    my $project = "dsci";
    my $dispatcher = Sparky::JobApi.new( :$job-id, :$project );

    mkdir "{%*ENV<HOME>}/artifacts";

    if $artifacts {
      say "load artifacts from previous jobs ...";
      for $artifacts<> -> $i {
        say "load $i ..."; 
        my $data = $dispatcher.get-file($i);
        $i.IO.spurt($data);
        move($i,"{%*ENV<HOME>}/artifacts/$i");
        say "{%*ENV<HOME>}/artifacts/$i OK"; 
      }
    }

    if tags()<plugin> {

      my $params = $me.get-stash()<params>;

      if tags()<dump-task-code> {
        %*ENV<SP6_DUMP_TASK_CODE> =  1;
      }

      if tags()<dump-task-config> {
        %*ENV<SP6_DUMP_TASK_CONFIG> =  1;
      }

      # inject git repo parameters
      
      $params<DSCI_COMMIT> = tags()<sha>;
      $params<DSCI_SCM> = tags()<scm>;
      $params<DSCI_JOB_ID> = tags()<job_id>;
      $params<DSCI_REF> = tags()<ref>;
      $params<DSCI_REPO_FULL_NAME> = tags()<repo_full_name>;
      $params<DSCI_MESSAGE> = tags()<message>;

      my $s = task-run "plugin: {tags()<plugin>}", tags()<plugin>, $params;

      $me.put-stash($s);

      return;

    } 

    directory "scm";

    git-scm tags()<scm>, %(
      :to<scm>,
      :branch(tags()<sha>),
      :skip-pull,
    );


    chdir "scm/.dsci/{tags()<path>}";

    bash q:to /BASH/, %( cwd => "{$*CWD}", description => "prepare dsci scripts" );
        find . -type f -name job.py  | raku -e  'for lines() -> $i { my $c = $i.IO.slurp; $c = "from sparrow6lib import *\n\n$c"; $i.IO.spurt($c) }' 
        find . -type f -name task.py  | raku -e  'for lines() -> $i { my $c = $i.IO.slurp; $c = "from sparrow6lib import *\n\n$c"; $i.IO.spurt($c) }' 
        find . -type f -name hook.py  | raku -e  'for lines() -> $i { my $c = $i.IO.slurp; $c = "from sparrow6lib import *\n\n$c"; $i.IO.spurt($c) }' 
    BASH

    my %p_ = $me.get-stash();

    my %params = %p_<params>; 

    %params<_dsci_> = %p_<dsci>;

    if tags()<dump-task-config> {
      %*ENV<SP6_DUMP_TASK_CONFIG> =  1;
    }

    if tags()<dump-task-code> {
      %*ENV<SP6_DUMP_TASK_CODE> =  1;
    }

    # inject git repo parameters
      
    %params<DSCI_COMMIT> = tags()<sha>;
    %params<DSCI_SCM> = tags()<scm>;
    %params<DSCI_JOB_ID> = tags()<job_id>;
    %params<DSCI_REF> = tags()<ref>;
    %params<DSCI_REPO_FULL_NAME> = tags()<repo_full_name>;
    %params<DSCI_MESSAGE> = tags()<message>;

    my $s = task-run ".", %params;

    if %*ENV<HOME> && %*ENV<HOME>.IO ~~ :d  && "{%*ENV<HOME>}/artifacts".IO ~~ :d {
      say "saving job artifacts ...";
      $s<artifacts> = [];
      for dir("{%*ENV<HOME>}/artifacts") -> $i {
        if $i.IO ~~ :f {
          $dispatcher.put-file($i.Str,$i.IO.basename);
          push $s<artifacts>, $i.IO.basename;
        }
      }
    }

    $me.put-stash($s);

  }

}

Pipeline.new.run;