From 83d3913ba738edf570f7753aa393a201cc26ca4e Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 23:07:14 +0800 Subject: [PATCH] feat: support more detailed problem config (#7) --- internal/api/runner/build.go | 14 +- internal/api/runner/judge.go | 32 ++-- internal/e/code.go | 2 + internal/service/runner/common.go | 41 ++--- internal/service/runner/compile.go | 53 +++--- internal/service/runner/config.go | 238 +++++++++++++++++-------- internal/service/runner/new_problem.go | 53 +++--- internal/service/runner/run_judge.go | 105 +++++------ internal/service/runner/service.go | 10 +- internal/service/runner/status.go | 18 +- 10 files changed, 330 insertions(+), 236 deletions(-) diff --git a/internal/api/runner/build.go b/internal/api/runner/build.go index 5d4348b..7cc13d5 100644 --- a/internal/api/runner/build.go +++ b/internal/api/runner/build.go @@ -6,6 +6,7 @@ import ( "fmt" "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/internal/model" + "git.0x7f.app/WOJ/woj-server/internal/service/runner" "github.com/hibiken/asynq" "go.uber.org/zap" "time" @@ -18,6 +19,7 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { } h.log.Debug("build", zap.Any("payload", p)) + meta := runner.JudgeMeta{Version: p.ProblemVersionID} status, ctx := func() (e.Status, string) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) @@ -25,16 +27,20 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { return e.InternalError, "{\"Message\": \"storage error\"}" } - config, status := h.runnerService.NewProblem(p.ProblemVersionID, url, true) + config, status := h.runnerService.NewProblem(&meta, url, true) if status != e.Success { return e.InternalError, "{\"Message\": \"build error: " + status.String() + "\"}" } for i := range config.Languages { // do not store in db - config.Languages[i].Type = "" - config.Languages[i].Script = "" - config.Languages[i].Cmp = "" + config.Languages[i].Judge.Type = "" + config.Languages[i].Judge.Script = "" + config.Languages[i].Judge.Cmp = "" + + // Compile and Run is ok + config.Languages[i].Runtime.Prebuild = runner.ConfigRuntime{} + config.Languages[i].Runtime.Check = runner.ConfigRuntime{} } b, _ := json.Marshal(config) diff --git a/internal/api/runner/judge.go b/internal/api/runner/judge.go index 83a2268..5e015d0 100644 --- a/internal/api/runner/judge.go +++ b/internal/api/runner/judge.go @@ -23,50 +23,52 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { user := utils.RandomString(16) h.log.Debug("judge", zap.Any("payload", p), zap.String("user", user)) + meta := runner.JudgeMeta{Version: p.ProblemVersionID, User: user, Lang: p.Submission.Language} - status, point, ctx := func() (e.Status, int32, runner.JudgeStatus) { - systemError := runner.JudgeStatus{ - Message: "System Error", - Tasks: []runner.TaskStatus{{Verdict: runner.VerdictSystemError, Message: "API Error"}}, - } + SystemError := &runner.JudgeStatus{ + Message: "System Error", + Tasks: []runner.TaskStatus{{Verdict: runner.VerdictSystemError, Message: "API Error"}}, + } + + status, point, ctx := func() (e.Status, int32, *runner.JudgeStatus) { // 1. write user code userCode := filepath.Join(runner.UserDir, user, fmt.Sprintf("%s.%s", user, p.Submission.Language)) if !file.Touch(userCode) { - return e.InternalError, 0, systemError + return e.InternalError, 0, SystemError } err := file.Write(userCode, []byte(p.Submission.Code)) if err != nil { - return e.InternalError, 0, systemError + return e.InternalError, 0, SystemError } // 2. check problem - if !h.runnerService.ProblemExists(p.ProblemVersionID) { + if !h.runnerService.ProblemExists(&meta) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) if status != e.Success { - return status, 0, systemError + return status, 0, SystemError } - _, status = h.runnerService.NewProblem(p.ProblemVersionID, url, false) + _, status = h.runnerService.NewProblem(&meta, url, false) if status != e.Success { - return status, 0, systemError + return status, 0, SystemError } } // 3. compile - compileResult, status := h.runnerService.Compile(p.ProblemVersionID, user, p.Submission.Language) + compileResult, status := h.runnerService.Compile(&meta) if status != e.Success { return status, 0, compileResult } // 4. run and judge - result, point, status := h.runnerService.RunAndJudge(p.ProblemVersionID, user, p.Submission.Language) + result, point, status := h.runnerService.RunAndJudge(&meta) return status, point, result }() if status == e.InternalError { // notice asynq to retry - return fmt.Errorf("internal error, ctx: %v", ctx) + return fmt.Errorf("internal error, ctx: %v", *ctx) } h.taskService.SubmitUpdate(&model.SubmitUpdatePayload{ @@ -75,7 +77,7 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { ProblemVersionID: p.ProblemVersionID, UserDir: user, Point: point, - }, ctx) + }, *ctx) return nil } diff --git a/internal/e/code.go b/internal/e/code.go index 9fa3e4e..78ef5c6 100644 --- a/internal/e/code.go +++ b/internal/e/code.go @@ -63,6 +63,7 @@ const ( RunnerUserCompileFailed RunnerRunFailed RunnerJudgeFailed + RunnerLanguageNotSupported ) const ( @@ -123,6 +124,7 @@ var msgText = map[Status]string{ RunnerUserCompileFailed: "Runner User Compile Failed", RunnerRunFailed: "Runner Run Failed", RunnerJudgeFailed: "Runner Judge Failed", + RunnerLanguageNotSupported: "Runner Language Not Supported", StorageUploadFailed: "Storage Upload Failed", StorageGetFailed: "Storage Get Failed", diff --git a/internal/service/runner/common.go b/internal/service/runner/common.go index 0f523ad..e76abc7 100644 --- a/internal/service/runner/common.go +++ b/internal/service/runner/common.go @@ -22,6 +22,12 @@ const ( ContainerImageRun = "git.0x7f.app/woj/ubuntu-run:latest" ) +type JudgeMeta struct { + Version uint + User string + Lang string +} + func init() { wd, err := os.Getwd() if err != nil { @@ -34,39 +40,34 @@ func init() { TmpDir = path.Join(Prefix, TmpDir) } -func (s *service) ProblemExists(version uint) bool { - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) +func (s *service) ProblemExists(meta *JudgeMeta) bool { + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) return file.Exist(problemPath) } -func (s *service) ValidatePath(version uint, user string, lang string) e.Status { - if !s.ProblemExists(version) { - s.log.Info("problem not exists", zap.Uint("version", version)) +func (s *service) ValidatePath(meta *JudgeMeta) e.Status { + if !s.ProblemExists(meta) { + s.log.Info("problem not exists", zap.Uint("version", meta.Version)) return e.RunnerProblemNotExist } - userPath := filepath.Join(UserDir, user, fmt.Sprintf("%s.%s", user, lang)) + userPath := filepath.Join(UserDir, meta.User, fmt.Sprintf("%s.%s", meta.User, meta.Lang)) if !file.Exist(userPath) { - s.log.Info("user program not exists", zap.String("user", user), zap.String("lang", lang)) + s.log.Info("user program not exists", zap.String("user", meta.User), zap.String("lang", meta.Lang)) return e.RunnerUserNotExist } return e.Success } -func (s *service) GetLangInfo(config *Config, lang string) (configLanguage, bool) { - for _, l := range config.Languages { - if l.Lang == lang { - return l, true - } +func (s *service) GetConfig(meta *JudgeMeta, skipCheck bool) (*Config, *ConfigLanguage, e.Status) { + config, err := s.ParseConfig(meta, skipCheck) + if err != nil { + return nil, nil, e.RunnerProblemParseFailed } - return configLanguage{}, false -} - -func (s *service) GetLangScript(l *configLanguage, lang string) string { - if l.Type == "default" { - return "/woj/framework/template/default/" + lang + ".Makefile" - } else { - return "/woj/problem/judge/" + l.Script + cLang, ok := config.FilterLanguage(meta.Lang) + if !ok { + return nil, nil, e.RunnerLanguageNotSupported } + return config, cLang, e.Success } diff --git a/internal/service/runner/compile.go b/internal/service/runner/compile.go index 919c4ff..5035dbe 100644 --- a/internal/service/runner/compile.go +++ b/internal/service/runner/compile.go @@ -13,34 +13,34 @@ import ( "time" ) -func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, e.Status) { +func (s *service) Compile(meta *JudgeMeta) (*JudgeStatus, e.Status) { // 1. ensure problem/user exists - status := s.ValidatePath(version, user, lang) + status := s.ValidatePath(meta) if status != e.Success { - return JudgeStatus{Message: "check failed"}, status + return &JudgeStatus{Message: "check failed"}, status } - config, err := s.ParseConfig(version, true) - if err != nil { - s.log.Error("[compile] parse config failed", zap.Error(err)) - return JudgeStatus{ + config, cLang, status := s.GetConfig(meta, true) + if status != e.Success { + s.log.Error("[compile] parse config failed", zap.Any("meta", *meta)) + return &JudgeStatus{ Message: "parse config failed", Tasks: []TaskStatus{{Verdict: VerdictSystemError, Message: "parse config failed"}}, }, e.RunnerProblemParseFailed } // 2. prepare judge environment - workDir := filepath.Join(UserDir, user) - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") + workDir := filepath.Join(UserDir, meta.User) + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") - sourceFile := filepath.Join(workDir, fmt.Sprintf("%s.%s", user, lang)) - targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", user)) + sourceFile := filepath.Join(workDir, fmt.Sprintf("%s.%s", meta.User, meta.Lang)) + targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", meta.User)) - logFile := filepath.Join(workDir, fmt.Sprintf("%s.compile.log", user)) + logFile := filepath.Join(workDir, fmt.Sprintf("%s.compile.log", meta.User)) log, err := os.Create(logFile) if err != nil { s.log.Error("[compile] create log file failed", zap.Error(err)) - return JudgeStatus{ + return &JudgeStatus{ Message: "create log file failed", Tasks: []TaskStatus{{Verdict: VerdictSystemError, Message: "create log file failed"}}, }, e.RunnerUserCompileFailed @@ -54,28 +54,28 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, DoAny(func() error { return os.Remove(targetFile) }). Do(func() error { return file.TouchErr(targetFile) }). Do(func() error { - l, ok := s.GetLangInfo(&config, lang) - script := s.GetLangScript(&l, lang) + l, ok := config.FilterLanguage(meta.Lang) if !ok { return e.RunnerProblemParseFailed.AsError() } + script := l.JudgeScript() args := &RunArgs{ Program: ProgramArgs{ Args: []string{"sh", "-c", fmt.Sprintf("cd /woj/user && make -f %s compile", script)}, - Env: []string{fmt.Sprintf("USER_PROG=%s", user), fmt.Sprintf("LANG=%s", lang)}, + Env: []string{fmt.Sprintf("USER_PROG=%s", meta.User), fmt.Sprintf("LANG=%s", meta.Lang)}, }, Runtime: RuntimeArgs{ Image: ContainerImageFull, - Memory: 256 * 1024 * 1024, // 256 MB - Timeout: time.Minute, + Pid: int64(cLang.Runtime.Compile.NProcLimit + 2), // bash + make + Memory: uint64(cLang.Runtime.Compile.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Compile.TimeLimit+1000)/1000) * time.Second, }, IO: IOArgs{ Output: log, Limit: 4 * 1024, // 4 KB }, } - args.Runtime.Mount = []specs.Mount{ { Source: judgeDir, @@ -85,13 +85,13 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, }, { Source: sourceFile, - Destination: fmt.Sprintf("/woj/user/%s.%s", user, lang), + Destination: fmt.Sprintf("/woj/user/%s.%s", meta.User, meta.Lang), Type: "bind", Options: []string{"rbind", "ro"}, }, { Source: targetFile, - Destination: fmt.Sprintf("/woj/user/%s.out", user), + Destination: fmt.Sprintf("/woj/user/%s.out", meta.User), Type: "bind", Options: []string{"rbind"}, }, @@ -102,12 +102,7 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, Done() if err != nil { - s.log.Info("[compile] compile failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[compile] compile failed", zap.Error(err), zap.Any("meta", *meta)) status = e.RunnerUserCompileFailed } @@ -118,11 +113,11 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, msgText := string(msg) if !file.Exist(targetFile) || file.Empty(targetFile) { - return JudgeStatus{ + return &JudgeStatus{ Message: "compile failed", Tasks: []TaskStatus{{Verdict: VerdictCompileError, Message: msgText}}}, utils.If(status == e.Success, e.RunnerUserCompileFailed, status) } - return JudgeStatus{}, e.Success + return &JudgeStatus{}, e.Success } diff --git a/internal/service/runner/config.go b/internal/service/runner/config.go index d51ac3a..92cd0cd 100644 --- a/internal/service/runner/config.go +++ b/internal/service/runner/config.go @@ -4,106 +4,146 @@ import ( "encoding/json" "errors" "fmt" + "git.0x7f.app/WOJ/woj-server/pkg/utils" "os" "path/filepath" ) -type configLanguage struct { - Lang string `json:"Lang"` - Type string `json:"Type,omitempty"` - Script string `json:"Script,omitempty"` - Cmp string `json:"Cmp,omitempty"` +type ConfigRuntime struct { + TimeLimit int `json:"TimeLimit"` + MemoryLimit int `json:"MemoryLimit"` + NProcLimit int `json:"NProcLimit"` +} + +var ( + DefaultPrebuildRuntime = ConfigRuntime{ + TimeLimit: 300000, + MemoryLimit: 256, + NProcLimit: 64, + } + DefaultCompileRuntime = ConfigRuntime{ + TimeLimit: 60000, + MemoryLimit: 256, + NProcLimit: 64, + } + DefaultCheckRuntime = ConfigRuntime{ + TimeLimit: 60000, + MemoryLimit: 128, + NProcLimit: 64, + } +) + +func (c *ConfigRuntime) Validate() error { + if c.TimeLimit <= 0 { + return errors.New("time limit <= 0") + } + if c.MemoryLimit <= 0 { + return errors.New("memory limit <= 0") + } + if c.NProcLimit <= 0 { + return errors.New("nproc limit <= 0") + } + return nil +} + +type ConfigJudge struct { + Type string `json:"Type"` + Script string `json:"Script"` + Cmp string `json:"Cmp"` +} + +func (c *ConfigJudge) Validate(base string, lang string) error { + if c.Type != "custom" && c.Type != "default" { + return fmt.Errorf("language %s has invalid type %s", lang, c.Type) + } + + if c.Type == "custom" { + if c.Script == "" { + return fmt.Errorf("language %s has empty script", lang) + } + + file := filepath.Join(base, "judge", c.Script) + _, err := os.Stat(file) + if err != nil { + return fmt.Errorf("language %s has invalid script %s", lang, c.Script) + } + } + + if c.Type == "default" { + if c.Cmp == "" { + return fmt.Errorf("language %s has empty cmp", lang) + } + } + + return nil +} + +type ConfigLanguage struct { + Lang string `json:"Lang"` + Judge ConfigJudge `json:"Judge"` + Runtime struct { + Prebuild ConfigRuntime `json:"Prebuild"` + Compile ConfigRuntime `json:"Compile"` + Run ConfigRuntime `json:"Run"` + Check ConfigRuntime `json:"Check"` + } `json:"Runtime"` +} + +func (l *ConfigLanguage) JudgeScript() string { + if l.Judge.Type == "default" { + return "/woj/framework/template/default/" + l.Lang + ".Makefile" + } else { + return "/woj/problem/judge/" + l.Judge.Script + } } type Config struct { - Runtime struct { - TimeLimit int `json:"TimeLimit"` - MemoryLimit int `json:"MemoryLimit"` - NProcLimit int `json:"NProcLimit"` - } `json:"Runtime"` - Languages []configLanguage `json:"Languages"` + Languages []ConfigLanguage `json:"Languages"` Tasks []struct { Id int `json:"Id"` Points int32 `json:"Points"` } `json:"Tasks"` } -func (s *service) ParseConfig(version uint, skipCheck bool) (Config, error) { - base := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) - file := filepath.Join(base, "config.json") - - data, err := os.ReadFile(file) - if err != nil { - return Config{}, err - } - - config := Config{} - err = json.Unmarshal(data, &config) - if err != nil { - return Config{}, err - } - - if skipCheck { - return config, nil - } - - err = s.ValidateConfig(&config, base) - if err != nil { - return Config{}, err - } - - return config, nil -} - -func (s *service) ValidateConfig(config *Config, base string) error { - if config.Runtime.TimeLimit < 0 { - return errors.New("time limit is negative") - } - if config.Runtime.MemoryLimit < 0 { - return errors.New("memory limit is negative") - } - if config.Runtime.NProcLimit < 0 { - return errors.New("nproc limit is negative") - } - +func (c *Config) Validate(base string) error { allowedLang := map[string]struct{}{ - "c": {}, - "cpp": {}, + "c": {}, + "cpp": {}, + "go": {}, + "rust": {}, + "python3": {}, + "pypy3": {}, } - for _, lang := range config.Languages { + + for _, lang := range c.Languages { + // check language if _, ok := allowedLang[lang.Lang]; !ok { return fmt.Errorf("language %s is not allowed", lang.Lang) } - if lang.Type != "custom" && lang.Type != "default" { - return fmt.Errorf("language %s has invalid type %s", lang.Lang, lang.Type) + err := utils.NewMust(). + // check judge config + Do(func() error { return lang.Judge.Validate(base, lang.Lang) }). + // check runtime limits + Do(func() error { return lang.Runtime.Prebuild.Validate() }). + Do(func() error { return lang.Runtime.Compile.Validate() }). + Do(func() error { return lang.Runtime.Run.Validate() }). + Do(func() error { return lang.Runtime.Check.Validate() }). + Done() + + if err != nil { + return err } - if lang.Type == "custom" { - if lang.Script == "" { - return fmt.Errorf("language %s has empty script", lang.Lang) - } - - file := filepath.Join(base, "judge", lang.Script) - _, err := os.Stat(file) - if err != nil { - return fmt.Errorf("language %s has invalid script %s", lang.Lang, lang.Script) - } - } - - if lang.Type == "default" { - if lang.Cmp == "" { - return fmt.Errorf("language %s has empty cmp", lang.Lang) - } - } } - if len(config.Tasks) == 0 { + if len(c.Tasks) == 0 { return errors.New("no tasks") } + ids := map[int]struct{}{} - total := (1 + len(config.Tasks)) * len(config.Tasks) / 2 - for _, task := range config.Tasks { + total := (1 + len(c.Tasks)) * len(c.Tasks) / 2 + for _, task := range c.Tasks { if task.Id <= 0 { return fmt.Errorf("task %d has non-positive id", task.Id) } @@ -125,3 +165,53 @@ func (s *service) ValidateConfig(config *Config, base string) error { return nil } + +func (c *Config) FilterLanguage(lang string) (*ConfigLanguage, bool) { + for idx := range c.Languages { + if c.Languages[idx].Lang == lang { + return &c.Languages[idx], true + } + } + return &ConfigLanguage{}, false +} + +func (s *service) ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) { + base := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) + file := filepath.Join(base, "config.json") + + data, err := os.ReadFile(file) + if err != nil { + return &Config{}, err + } + + // unmarshal + config := &Config{} + err = json.Unmarshal(data, config) + if err != nil { + return &Config{}, err + } + + // fill default + for idx := range config.Languages { + if config.Languages[idx].Runtime.Prebuild.Validate() != nil { + config.Languages[idx].Runtime.Prebuild = DefaultPrebuildRuntime + } + if config.Languages[idx].Runtime.Compile.Validate() != nil { + config.Languages[idx].Runtime.Compile = DefaultCompileRuntime + } + if config.Languages[idx].Runtime.Check.Validate() != nil { + config.Languages[idx].Runtime.Check = DefaultCheckRuntime + } + } + + if skipCheck { + return config, nil + } + + err = config.Validate(base) + if err != nil { + return &Config{}, err + } + + return config, nil +} diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index 687002e..d75ec28 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -13,9 +13,9 @@ import ( "time" ) -func (s *service) DownloadProblem(version uint, url string) e.Status { - zipPath := filepath.Join(TmpDir, fmt.Sprintf("%d.zip", version)) - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) +func (s *service) DownloadProblem(meta *JudgeMeta, url string) e.Status { + zipPath := filepath.Join(TmpDir, fmt.Sprintf("%d.zip", meta.Version)) + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) err := down.Down(zipPath, url) if err != nil { @@ -32,26 +32,32 @@ func (s *service) DownloadProblem(version uint, url string) e.Status { return e.Success } -func (s *service) PrebuildProblem(version uint, force bool) e.Status { - if !s.ProblemExists(version) { +func (s *service) PrebuildProblem(meta *JudgeMeta, force bool) e.Status { + if !s.ProblemExists(meta) { return e.RunnerProblemNotExist } - mark := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), ".mark.prebuild") + mark := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), ".mark.prebuild") if force { _ = os.Remove(mark) } else if file.Exist(mark) { return e.Success } - prebuildScript := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge", "prebuild.Makefile") + prebuildScript := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge", "prebuild.Makefile") if !file.Exist(prebuildScript) { - s.log.Info("[new] prebuild script not found", zap.String("path", prebuildScript), zap.Uint("version", version)) + s.log.Info("[new] prebuild script not found", zap.String("path", prebuildScript), zap.Uint("version", meta.Version)) return e.Success } - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") + _, cLang, status := s.GetConfig(meta, false) + if status != e.Success { + s.log.Error("[new] parse config failed", zap.Any("meta", *meta)) + return e.RunnerProblemParseFailed + } + + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data") + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") args := &RunArgs{ Program: ProgramArgs{ @@ -59,8 +65,9 @@ func (s *service) PrebuildProblem(version uint, force bool) e.Status { }, Runtime: RuntimeArgs{ Image: ContainerImageFull, - Memory: 256 * 1024 * 1024, // 256MB - Timeout: 5 * time.Minute, + Pid: int64(cLang.Runtime.Prebuild.NProcLimit + 3), // sh + bash + make + Memory: uint64(cLang.Runtime.Prebuild.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Prebuild.TimeLimit+1000)/1000) * time.Second, }, } args.Runtime.Mount = []specs.Mount{ @@ -81,35 +88,35 @@ func (s *service) PrebuildProblem(version uint, force bool) e.Status { err := s.ContainerRun(args) if err != nil { - s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", version)) + s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", meta.Version)) return e.RunnerProblemPrebuildFailed } return e.Success } -func (s *service) NewProblem(version uint, url string, force bool) (Config, e.Status) { +func (s *service) NewProblem(meta *JudgeMeta, url string, force bool) (*Config, e.Status) { if force { - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) _ = os.RemoveAll(problemPath) } - if !s.ProblemExists(version) { - status := s.DownloadProblem(version, url) + if !s.ProblemExists(meta) { + status := s.DownloadProblem(meta, url) if status != e.Success { - return Config{}, status + return &Config{}, status } } - cfg, err := s.ParseConfig(version, false) + cfg, err := s.ParseConfig(meta, false) if err != nil { - s.log.Info("[new] parse problem failed", zap.Error(err), zap.Uint("version", version)) - return Config{}, e.RunnerProblemParseFailed + s.log.Info("[new] parse problem failed", zap.Error(err), zap.Uint("version", meta.Version)) + return &Config{}, e.RunnerProblemParseFailed } - status := s.PrebuildProblem(version, true) + status := s.PrebuildProblem(meta, true) if status != e.Success { - return Config{}, status + return &Config{}, status } return cfg, e.Success diff --git a/internal/service/runner/run_judge.go b/internal/service/runner/run_judge.go index c1cbf36..761f4f1 100644 --- a/internal/service/runner/run_judge.go +++ b/internal/service/runner/run_judge.go @@ -13,37 +13,43 @@ import ( "time" ) -func (s *service) SandboxArgsBuilder(config *Config, user string, lang string, id int) string { +func (s *service) SandboxArgsBuilder(meta *JudgeMeta, cLang *ConfigLanguage, id int) string { var args []string - args = append(args, fmt.Sprintf("--memory_limit=%d", config.Runtime.MemoryLimit)) - args = append(args, fmt.Sprintf("--nproc_limit=%d", config.Runtime.NProcLimit)) - args = append(args, fmt.Sprintf("--time_limit=%d", config.Runtime.TimeLimit)) - args = append(args, fmt.Sprintf("--sandbox_template=%s", lang)) + args = append(args, fmt.Sprintf("--memory_limit=%d", cLang.Runtime.Run.MemoryLimit)) + args = append(args, fmt.Sprintf("--nproc_limit=%d", cLang.Runtime.Run.NProcLimit)) + args = append(args, fmt.Sprintf("--time_limit=%d", cLang.Runtime.Run.TimeLimit)) + args = append(args, fmt.Sprintf("--sandbox_template=%s", cLang.Lang)) args = append(args, fmt.Sprintf("--sandbox_action=ret")) args = append(args, fmt.Sprintf("--uid=1000")) args = append(args, fmt.Sprintf("--gid=1000")) args = append(args, fmt.Sprintf("--file_input=/woj/problem/data/input/%d.input", id)) args = append(args, fmt.Sprintf("--file_output=/woj/user/%d.out.usr", id)) args = append(args, fmt.Sprintf("--file_info=/woj/user/%d.info", id)) - args = append(args, fmt.Sprintf("--program=/woj/user/%s.out", user)) + args = append(args, fmt.Sprintf("--program=/woj/user/%s.out", meta.User)) return strings.Join(args, " ") } -func (s *service) ProblemRun(version uint, user string, lang string, config *Config) { - workDir := filepath.Join(UserDir, user) - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data", "input") +func (s *service) ProblemRun(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) { + workDir := filepath.Join(UserDir, meta.User) + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data", "input") - // woj-sandbox killer will add 1 more second, here we add 1 also - timeout := time.Duration((config.Runtime.TimeLimit+1000)/1000+1+1) * time.Second + runtimeArgs := RuntimeArgs{ + Image: ContainerImageRun, + // sh, woj_launcher:program, woj_launcher:killer, woj_launcher:stat + Pid: int64(cLang.Runtime.Run.NProcLimit + 4), + Memory: uint64(cLang.Runtime.Run.MemoryLimit * 1024 * 1024), + // woj-sandbox killer will add 1 more second, here we add 1 also + Timeout: time.Duration((cLang.Runtime.Run.TimeLimit+1000)/1000+1+1) * time.Second, + } ids := make([]int, 0) for _, task := range config.Tasks { f := func(id int) func() { return func() { testCase := filepath.Join(dataDir, fmt.Sprintf("%d.input", id)) - targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", user)) + targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", meta.User)) ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) ifoFile := filepath.Join(workDir, fmt.Sprintf("%d.info", id)) @@ -52,16 +58,10 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con Args: []string{ "sh", "-c", "cd /woj/user && /woj/framework/scripts/woj_launcher " + - s.SandboxArgsBuilder(config, user, lang, id), + s.SandboxArgsBuilder(meta, cLang, id), }, }, - Runtime: RuntimeArgs{ - Image: ContainerImageRun, - // sh, woj_launcher:program, woj_launcher:killer, woj_launcher:stat - Pid: int64(config.Runtime.NProcLimit + 4), - Memory: uint64(config.Runtime.MemoryLimit * 1024 * 1024), - Timeout: timeout, - }, + Runtime: runtimeArgs, } args.Runtime.Mount = []specs.Mount{ { @@ -72,7 +72,7 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con }, { Source: targetFile, - Destination: fmt.Sprintf("/woj/user/%s.out", user), + Destination: fmt.Sprintf("/woj/user/%s.out", meta.User), Type: "bind", Options: []string{"rbind", "ro"}, }, @@ -99,12 +99,7 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con Done() if err != nil { - s.log.Info("[run] run failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[run] run failed", zap.Error(err), zap.Any("meta", *meta)) } } }(task.Id) @@ -118,10 +113,18 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con } } -func (s *service) ProblemJudge(version uint, user string, lang string, config *Config) { - workDir := filepath.Join(UserDir, user) - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") +func (s *service) ProblemJudge(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) { + workDir := filepath.Join(UserDir, meta.User) + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data") + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") + script := cLang.JudgeScript() + + runtimeArgs := RuntimeArgs{ + Image: ContainerImageFull, + Pid: int64(cLang.Runtime.Check.NProcLimit + 2), // bash + make + Memory: uint64(cLang.Runtime.Check.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Check.TimeLimit+1000)/1000) * time.Second, + } ids := make([]int, 0) for _, task := range config.Tasks { @@ -130,25 +133,18 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) jdgFile := filepath.Join(workDir, fmt.Sprintf("%d.judge", id)) - c, ok := s.GetLangInfo(config, lang) - if !ok { - return - } - args := &RunArgs{ Program: ProgramArgs{ Args: []string{ "sh", "-c", - fmt.Sprintf("cd /woj/user && make -f %s judge", s.GetLangScript(&c, lang)), + fmt.Sprintf("cd /woj/user && make -f %s judge", script), }, Env: []string{ fmt.Sprintf("TEST_NUM=%d", id), - fmt.Sprintf("CMP=%s", c.Cmp), + fmt.Sprintf("CMP=%s", cLang.Judge.Cmp), }, }, - Runtime: RuntimeArgs{ - Image: ContainerImageFull, - }, + Runtime: runtimeArgs, } args.Runtime.Mount = []specs.Mount{ { @@ -184,12 +180,7 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C Done() if err != nil { - s.log.Info("[judge] judge failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[judge] judge failed", zap.Error(err), zap.Any("meta", *meta)) } } }(task.Id) @@ -203,27 +194,27 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C } } -func (s *service) RunAndJudge(version uint, user string, lang string) (JudgeStatus, int32, e.Status) { +func (s *service) RunAndJudge(meta *JudgeMeta) (*JudgeStatus, int32, e.Status) { // 1. ensure problem/user exists - status := s.ValidatePath(version, user, lang) + status := s.ValidatePath(meta) if status != e.Success { - return JudgeStatus{Message: "check failed"}, 0, status + return &JudgeStatus{Message: "check failed"}, 0, status } // 2. config - config, err := s.ParseConfig(version, false) - if err != nil { - return JudgeStatus{Message: "parse config failed"}, 0, e.RunnerProblemParseFailed + config, cLang, status := s.GetConfig(meta, true) + if status != e.Success { + return &JudgeStatus{Message: status.String()}, 0, status } // 3. run user program - s.ProblemRun(version, user, lang, &config) + s.ProblemRun(meta, config, cLang) - // 4. run judger - s.ProblemJudge(version, user, lang, &config) + // 4. run judge + s.ProblemJudge(meta, config, cLang) // 5. check result - result, pts := s.checkResults(user, &config) + result, pts := s.CheckResults(meta, config, cLang) return result, pts, e.Success } diff --git a/internal/service/runner/service.go b/internal/service/runner/service.go index ab5c826..fca0f9d 100644 --- a/internal/service/runner/service.go +++ b/internal/service/runner/service.go @@ -21,17 +21,17 @@ type Service interface { // EnsureDeps build docker images EnsureDeps(force bool) e.Status // NewProblem = Download + Parse + Prebuild - NewProblem(version uint, url string, force bool) (Config, e.Status) + NewProblem(meta *JudgeMeta, url string, force bool) (*Config, e.Status) // Compile compile user submission - Compile(version uint, user string, lang string) (JudgeStatus, e.Status) + Compile(meta *JudgeMeta) (*JudgeStatus, e.Status) // RunAndJudge execute user program - RunAndJudge(version uint, user string, lang string) (JudgeStatus, int32, e.Status) + RunAndJudge(meta *JudgeMeta) (*JudgeStatus, int32, e.Status) // ParseConfig parse config file - ParseConfig(version uint, skipCheck bool) (Config, error) + ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) // ProblemExists check if problem exists - ProblemExists(version uint) bool + ProblemExists(meta *JudgeMeta) bool HealthCheck() error Shutdown() error diff --git a/internal/service/runner/status.go b/internal/service/runner/status.go index 95ddfd8..c800760 100644 --- a/internal/service/runner/status.go +++ b/internal/service/runner/status.go @@ -97,12 +97,12 @@ func (t *TaskStatus) checkExit() *TaskStatus { return t } -func (t *TaskStatus) checkTime(config *Config) *TaskStatus { +func (t *TaskStatus) checkTime(cLang *ConfigLanguage) *TaskStatus { if t.Verdict != VerdictAccepted { return t } - if t.info["real_time"].(float64) > float64(config.Runtime.TimeLimit)+5 { + if t.info["real_time"].(float64) > float64(cLang.Runtime.Run.TimeLimit)+5 { t.Verdict = VerdictTimeLimitExceeded t.Message = fmt.Sprintf("real_time: %v cpu_time: %v", t.info["real_time"], t.info["cpu_time"]) } @@ -110,12 +110,12 @@ func (t *TaskStatus) checkTime(config *Config) *TaskStatus { return t } -func (t *TaskStatus) checkMemory(config *Config) *TaskStatus { +func (t *TaskStatus) checkMemory(cLang *ConfigLanguage) *TaskStatus { if t.Verdict != VerdictAccepted { return t } - if t.info["memory"].(float64) > float64((config.Runtime.MemoryLimit+1)*1024) { + if t.info["memory"].(float64) > float64((cLang.Runtime.Run.MemoryLimit+1)*1024) { t.Verdict = VerdictMemoryLimitExceeded t.Message = fmt.Sprintf("memory: %v", t.info["memory"]) } @@ -193,7 +193,7 @@ func (t *TaskStatus) checkJudge(pts *map[int]int32) *TaskStatus { return t } -func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) { +func (s *service) CheckResults(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) (*JudgeStatus, int32) { // CE will be processed in phase compile pts := map[int]int32{} @@ -202,7 +202,7 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) } var results []TaskStatus - dir := filepath.Join(UserDir, user) + dir := filepath.Join(UserDir, meta.User) var sum int32 = 0 for i := 1; i <= len(config.Tasks); i++ { @@ -213,8 +213,8 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) result.getInfoText(info). getInfo(). - checkTime(config). - checkMemory(config). + checkTime(cLang). + checkMemory(cLang). checkExit(). getJudgeText(judge). getJudge(). @@ -224,5 +224,5 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) results = append(results, result) } - return JudgeStatus{Message: "", Tasks: results}, sum + return &JudgeStatus{Message: "", Tasks: results}, sum }