diff --git a/internal/service/runner/common.go b/internal/service/runner/common.go index b0de8c3..e1aa973 100644 --- a/internal/service/runner/common.go +++ b/internal/service/runner/common.go @@ -6,7 +6,6 @@ import ( "git.0x7f.app/WOJ/woj-server/pkg/file" "go.uber.org/zap" "os" - "os/exec" "path" "path/filepath" ) @@ -27,21 +26,10 @@ func init() { Prefix = path.Join(wd, Prefix) ProblemDir = path.Join(Prefix, ProblemDir) - ScriptsDir = path.Join(Prefix, ScriptsDir) UserDir = path.Join(Prefix, UserDir) TmpDir = path.Join(Prefix, TmpDir) } -func (s *service) execute(exe string, args ...string) error { - cmd := exec.Command(exe, args...) - cmd.Dir = Prefix - if s.verbose { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } - return cmd.Run() -} - func (s *service) checkAndExecute(version uint, user string, lang string, script string, fail e.Status) e.Status { if !s.ProblemExists(version) { s.log.Info("problem not exists", zap.Uint("version", version)) diff --git a/internal/service/runner/exec.go b/internal/service/runner/exec.go new file mode 100644 index 0000000..58e5bb8 --- /dev/null +++ b/internal/service/runner/exec.go @@ -0,0 +1,109 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "git.0x7f.app/WOJ/woj-server/pkg/file" + "git.0x7f.app/WOJ/woj-server/pkg/utils" + "os" + "os/exec" + "time" +) + +func (s *service) execute(exe string, args ...string) error { + cmd := exec.Command(exe, args...) + cmd.Dir = Prefix + if s.verbose { + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } + return cmd.Run() +} + +type executeArgs struct { + exe string + args []string + + timeout time.Duration + kill func() error + + output *os.File + limit int64 +} + +func (s *service) executeTimeout(arg *executeArgs) error { + ctx, cancel := context.WithTimeout(context.Background(), arg.timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, arg.exe, arg.args...) + cmd.Dir = Prefix + if arg.kill != nil { + cmd.Cancel = arg.kill + } + if s.verbose && arg.output == nil { + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + } else if arg.output != nil { + if arg.limit == 0 { + cmd.Stdout = arg.output + cmd.Stderr = arg.output + } else { + lw := &file.LimitedWriter{ + File: arg.output, + Limit: arg.limit, + } + cmd.Stdout = lw + cmd.Stderr = lw + } + } + + err := cmd.Start() + if err != nil { + return err + } + + err = cmd.Wait() + + // make sure the process is killed + _ = cmd.Process.Kill() + + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return fmt.Errorf("command timed out") + } + + return err +} + +type podmanArgs struct { + executeArgs + memory string +} + +func (s *service) podmanRun(arg *podmanArgs) error { + name := fmt.Sprintf("woj-%d-%s", time.Now().UnixNano(), utils.RandomString(8)) + execArgs := &executeArgs{ + exe: "podman", + output: utils.If(arg.output == nil, os.Stderr, arg.output), + limit: utils.If(arg.limit == 0, 4*1024, arg.limit), + timeout: utils.If(arg.timeout == 0, 10*time.Second, arg.timeout), + kill: func() error { + if arg.kill != nil { + _ = arg.kill() + } + return s.execute("podman", "kill", name) + }, + } + + args := []string{ + "run", + "--rm", + "--name", name, + "--memory", utils.If(arg.memory == "", "256m", arg.memory), + } + args = append(args, arg.args...) + + execArgs.args = args + + return s.executeTimeout(execArgs) +} diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index f4b025e..ec8b3de 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -6,10 +6,10 @@ import ( "git.0x7f.app/WOJ/woj-server/pkg/down" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/unzip" - "git.0x7f.app/WOJ/woj-server/pkg/utils" "go.uber.org/zap" "os" "path/filepath" + "time" ) func (s *service) download(version uint, url string) e.Status { @@ -43,7 +43,30 @@ func (s *service) prebuild(version uint, force bool) e.Status { return e.Success } - err := s.execute("problem_prebuild.sh", fmt.Sprintf("%d", version)) + prebuildScript := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge", "prebuild.Makefile") + if !file.FileExist(prebuildScript) { + s.log.Info("prebuild script not found", zap.String("path", prebuildScript), zap.Uint("version", version)) + return e.Success + } + + dataPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") + judgePath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") + + args := []string{ + "-v", dataPath + ":/woj/problem/data", + "-v", judgePath + ":/woj/problem/judge", + "-e", "PREFIX=/woj/problem", + "git.0x7f.app/woj/ubuntu-full:latest", + "sh", "-c", "cd /woj/problem/judge && make -f prebuild.Makefile prebuild && touch .mark.prebuild", + } + runArgs := &podmanArgs{ + executeArgs: executeArgs{ + args: args, + timeout: 300 * time.Second, + }, + memory: "1g", + } + err := s.podmanRun(runArgs) if err != nil { s.log.Warn("prebuild problem failed", zap.Error(err), zap.Uint("version", version)) @@ -68,6 +91,7 @@ func (s *service) NewProblem(version uint, url string, force bool) (Config, e.St cfg, err := s.ParseConfig(version, false) if err != nil { + // TODO: err is dropped here, should return to server: add a new column in problem_version table return Config{}, e.RunnerProblemParseFailed } diff --git a/resource/runner/scripts/problem_prebuild.sh b/resource/runner/scripts/problem_prebuild.sh deleted file mode 100755 index 7bf9d3e..0000000 --- a/resource/runner/scripts/problem_prebuild.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -WORKSPACE=$(cd "$(dirname "$0")"/.. && pwd) -. "$WORKSPACE"/scripts/common.sh -. "$WORKSPACE"/scripts/docker_run.sh - -if [ "$1" == "" ] || [ ! -d "$WORKSPACE/problem/$1" ]; then - log_warn "Usage: $0 " - exit 1 -fi - -if [ -f "$WORKSPACE/problem/$1/.mark.prebuild" ]; then - log_warn "Problem $1 already prebuilt" - log_warn "If you want to re-prebuild the problem, please remove the file $WORKSPACE/problem/$1/.mark.prebuild" - exit 0 -fi - -if [ ! -f "$WORKSPACE/problem/$1/judge/prebuild.Makefile" ]; then - log_warn "Problem $1 does not have prebuild scripts" - touch "$WORKSPACE/problem/$1/.mark.prebuild" - exit 0 -fi - -export TIMEOUT=${2:-300} -export MEMORY="1g" -docker_run \ - -v "$WORKSPACE/problem/$1/data":/woj/problem/data \ - -v "$WORKSPACE/problem/$1/judge":/woj/problem/judge \ - -e PREFIX=/woj/problem \ - git.0x7f.app/woj/ubuntu-full \ - sh -c "cd /woj/problem/judge && make -f prebuild.Makefile prebuild && touch .mark.prebuild" - -mv "$WORKSPACE/problem/$1/judge/.mark.prebuild" "$WORKSPACE/problem/$1/.mark.prebuild" || exit 1