From d1f72b758e9d97878b6dd21ca68762951c83b948 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 20:34:16 +0800 Subject: [PATCH 01/24] feat: also log user dir in status table --- internal/api/consumer/submitUpdate.go | 1 + internal/api/runner/judge.go | 1 + internal/model/Status.go | 1 + internal/model/Task.go | 1 + internal/service/status/create.go | 2 ++ 5 files changed, 6 insertions(+) diff --git a/internal/api/consumer/submitUpdate.go b/internal/api/consumer/submitUpdate.go index d016438..c54491a 100644 --- a/internal/api/consumer/submitUpdate.go +++ b/internal/api/consumer/submitUpdate.go @@ -24,6 +24,7 @@ func (h *handler) SubmitUpdate(_ context.Context, t *asynq.Task) error { createData := &status.CreateData{ SubmissionID: p.SubmissionID, ProblemVersionID: p.ProblemVersionID, + UserDir: p.UserDir, Context: p.Context, Point: p.Point, } diff --git a/internal/api/runner/judge.go b/internal/api/runner/judge.go index 8ea1bb5..511d8a4 100644 --- a/internal/api/runner/judge.go +++ b/internal/api/runner/judge.go @@ -68,6 +68,7 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { Status: status, SubmissionID: p.Submission.ID, ProblemVersionID: p.ProblemVersionID, + UserDir: user, Point: point, }, ctx) diff --git a/internal/model/Status.go b/internal/model/Status.go index 2e14cf1..ad1db38 100644 --- a/internal/model/Status.go +++ b/internal/model/Status.go @@ -10,6 +10,7 @@ type Status struct { SubmissionID uint `json:"-" gorm:"not null;index"` Submission Submission `json:"submission" gorm:"foreignKey:SubmissionID"` ProblemVersionID uint `json:"problem_version_id" gorm:"not null;index"` + UserDir string `json:"user_dir" gorm:"not null"` Context pgtype.JSON `json:"context" gorm:"type:json;not null"` Point int32 `json:"point" gorm:"not null"` IsEnabled bool `json:"is_enabled" gorm:"not null;index"` diff --git a/internal/model/Task.go b/internal/model/Task.go index 9b2cc05..9aea117 100644 --- a/internal/model/Task.go +++ b/internal/model/Task.go @@ -38,5 +38,6 @@ type SubmitUpdatePayload struct { SubmissionID uint ProblemVersionID uint Point int32 + UserDir string Context string } diff --git a/internal/service/status/create.go b/internal/service/status/create.go index e933dba..6121df6 100644 --- a/internal/service/status/create.go +++ b/internal/service/status/create.go @@ -10,6 +10,7 @@ import ( type CreateData struct { SubmissionID uint ProblemVersionID uint + UserDir string Context string Point int32 } @@ -18,6 +19,7 @@ func (s *service) Create(data *CreateData) (*model.Status, e.Status) { status := &model.Status{ SubmissionID: data.SubmissionID, ProblemVersionID: data.ProblemVersionID, + UserDir: data.UserDir, Context: pgtype.JSON{ Bytes: []byte(data.Context), Status: pgtype.Present, From 7f71fb4554d4ce5cce9ccb8d28e0b0d7b2fed31c Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:03:16 +0800 Subject: [PATCH 02/24] fix: query_one might crash if submitStatus is empty --- internal/api/status/query_one.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/api/status/query_one.go b/internal/api/status/query_one.go index 937f07c..9579f12 100644 --- a/internal/api/status/query_one.go +++ b/internal/api/status/query_one.go @@ -35,6 +35,9 @@ func (h *handler) QueryBySubmissionID(c *gin.Context) { // query status submitStatus, status := h.statusService.Query(req.SubmissionID, true) + if status != e.Success { + e.Pong[any](c, status, nil) + } // check permission role := claim.(*model.Claim).Role From b5d14d5a4b62bd82d27aa11e58d768ff657355ab Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:03:30 +0800 Subject: [PATCH 03/24] fix: deadlock in pool --- .gitignore | 1 + Makefile | 9 ++++++++- pkg/pool/pool.go | 9 +++++---- pkg/pool/pool_test.go | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index dc4048b..7050f75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /woj /config.yaml /dsn.txt +/coverage.* diff --git a/Makefile b/Makefile index ce25cb9..02d0810 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ LDFLAGS += -s -w GOBUILD := $(GO) build -ldflags '$(LDFLAGS)' GOBIN := $(shell go env GOPATH)/bin -.PHONY: all build clean dep swagger fmt +.PHONY: all build clean dep swagger fmt coverage test default: all @@ -35,3 +35,10 @@ swagger: fmt: go fmt ./... + +coverage: + go test ./... -coverprofile=coverage.out + go tool cover -html=coverage.out -o coverage.html + +test: + go test -race ./... diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index 1e8fb78..f1a04ca 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -33,17 +33,18 @@ func (tp *TaskPool) Start() { func (tp *TaskPool) AddTask(f func()) int { tp.lck.Lock() - defer tp.lck.Unlock() id := tp.curTaskID tp.curTaskID++ - task := Task{id: id, f: f} - tp.queue <- task - waitChan := make(chan struct{}) tp.waitMap[id] = waitChan + tp.lck.Unlock() + + task := Task{id: id, f: f} + tp.queue <- task + return id } diff --git a/pkg/pool/pool_test.go b/pkg/pool/pool_test.go index fe21f03..f82680c 100644 --- a/pkg/pool/pool_test.go +++ b/pkg/pool/pool_test.go @@ -60,3 +60,39 @@ func TestTaskPool_WaitForTask(t *testing.T) { pool.Stop() } + +func TestTaskPool_One(t *testing.T) { + pool := NewTaskPool(1, 1) + pool.Start() + + lck := sync.Mutex{} + counter := 0 + + ids := make([]int, 0) + for i := 1; i <= 10; i++ { + f := func(i int) func() { + return func() { + lck.Lock() + t.Log("task", i, "locked") + counter += i + t.Log("task", i, "unlocked") + lck.Unlock() + + time.Sleep(time.Duration(i*10) * time.Millisecond) + t.Log("task", i, "finished") + } + }(i) + id := pool.AddTask(f) + ids = append(ids, id) + } + + for _, id := range ids { + pool.WaitForTask(id) + } + + if counter != 55 { + t.Error("some tasks were not executed") + } + + pool.Stop() +} From ba7afb5ecaef226fb57c7dea8bad76c82be4a63e Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:04:20 +0800 Subject: [PATCH 04/24] chore: bump version to dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d2d61a7..0d9c5e7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.2 \ No newline at end of file +1.2.3-dev \ No newline at end of file From f5e2b0cf8b55abb43ca97d6b5fbebd9548f381e9 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:05:54 +0800 Subject: [PATCH 05/24] fix: adjust sentrygin position --- internal/web/router/router.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/web/router/router.go b/internal/web/router/router.go index b2c794d..a92f139 100644 --- a/internal/web/router/router.go +++ b/internal/web/router/router.go @@ -72,9 +72,6 @@ func (s *service) initRouters(conf *model.Config, injector *do.Injector) *gin.En // |Middlewares| // +-----------+ - // Sentry middleware - r.Use(sentrygin.New(sentrygin.Options{Repanic: true})) - // Logger middleware and debug if conf.Development { // Gin's default logger is pretty enough @@ -88,6 +85,9 @@ func (s *service) initRouters(conf *model.Config, injector *do.Injector) *gin.En r.Use(ginZap.RecoveryWithZap(ginLog, true)) } + // Sentry middleware + r.Use(sentrygin.New(sentrygin.Options{Repanic: true})) + // CORS middleware r.Use(cors.New(cors.Config{ AllowAllOrigins: true, From 7d95c88063dbffa965af52dba63de7b589a74e55 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:24:14 +0800 Subject: [PATCH 06/24] fix: query_one might crash if submitStatus is empty --- internal/api/status/query_one.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/api/status/query_one.go b/internal/api/status/query_one.go index 9579f12..665cadd 100644 --- a/internal/api/status/query_one.go +++ b/internal/api/status/query_one.go @@ -37,11 +37,13 @@ func (h *handler) QueryBySubmissionID(c *gin.Context) { submitStatus, status := h.statusService.Query(req.SubmissionID, true) if status != e.Success { e.Pong[any](c, status, nil) + return } // check permission role := claim.(*model.Claim).Role - if role >= model.RoleAdmin || submitStatus.Submission.UserID == claim.(*model.Claim).UID { + uid := claim.(*model.Claim).UID + if role >= model.RoleAdmin || submitStatus.Submission.UserID == uid { // full status e.Pong(c, status, submitStatus) return From 84a79c5460e0daa9a710fb637bde468f69388014 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:31:10 +0800 Subject: [PATCH 07/24] chore: use short commit hash --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 02d0810..290b653 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO := go PKG_BASE := $(shell head -n 1 go.mod | awk '{print $$2}') BUILD_TIME := $(shell date -u '+%Y%m%d-%I%M%S') VERSION := $(shell cat VERSION) -GIT_COMMIT := $(shell git rev-parse HEAD) +GIT_COMMIT := $(shell git rev-parse --short HEAD) LDFLAGS += -X $(PKG_BASE)/cmd.BuildTime=$(BUILD_TIME) LDFLAGS += -X $(PKG_BASE)/cmd.Version=$(VERSION) From c36d68eddde79071cd2ab8ca75222e48319b1654 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 6 Jan 2024 21:55:44 +0800 Subject: [PATCH 08/24] feat: metrics: do not log oauth callback --- internal/web/metrics/middleware.go | 9 +++++++++ internal/web/metrics/prometheus.go | 4 +++- internal/web/router/router.go | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/web/metrics/middleware.go b/internal/web/metrics/middleware.go index 36edd35..7690853 100644 --- a/internal/web/metrics/middleware.go +++ b/internal/web/metrics/middleware.go @@ -13,6 +13,10 @@ func (s *service) SetLogPaths(paths []string) { s.logPaths = paths } +func (s *service) SetIgnorePaths(paths []string) { + s.ignorePaths = paths +} + func (s *service) Handler() gin.HandlerFunc { return func(c *gin.Context) { url := c.Request.URL.String() @@ -23,6 +27,11 @@ func (s *service) Handler() gin.HandlerFunc { return } + if utils.Contains(s.ignorePaths, func(path string) bool { return strings.HasPrefix(url, path) }) { + c.Next() + return + } + start := time.Now() c.Next() elapsed := float64(time.Since(start)) / float64(time.Millisecond) diff --git a/internal/web/metrics/prometheus.go b/internal/web/metrics/prometheus.go index ad26169..20d8c0e 100644 --- a/internal/web/metrics/prometheus.go +++ b/internal/web/metrics/prometheus.go @@ -14,6 +14,7 @@ var _ Service = (*service)(nil) type Service interface { Record(method, url string, success bool, httpCode int, errCode e.Status, elapsed float64) SetLogPaths(paths []string) + SetIgnorePaths(paths []string) Handler() gin.HandlerFunc HealthCheck() error } @@ -34,7 +35,8 @@ type service struct { counter *prometheus.CounterVec hist *prometheus.HistogramVec - logPaths []string + logPaths []string + ignorePaths []string } func (s *service) HealthCheck() error { diff --git a/internal/web/router/router.go b/internal/web/router/router.go index a92f139..8f9dfca 100644 --- a/internal/web/router/router.go +++ b/internal/web/router/router.go @@ -98,6 +98,7 @@ func (s *service) initRouters(conf *model.Config, injector *do.Injector) *gin.En // Prometheus middleware s.metric.SetLogPaths([]string{"/api"}) + s.metric.SetIgnorePaths([]string{"/api/v1/oauth/callback"}) r.Use(s.metric.Handler()) // +------+ From dd86ed8b5a1b4fb72181678f2899d1583bd6a346 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 7 Jan 2024 00:26:18 +0800 Subject: [PATCH 09/24] feat: LimitedWriter: write as much as possible --- internal/service/runner/exec.go | 2 ++ pkg/file/writer.go | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/internal/service/runner/exec.go b/internal/service/runner/exec.go index 58e5bb8..79dd522 100644 --- a/internal/service/runner/exec.go +++ b/internal/service/runner/exec.go @@ -6,6 +6,7 @@ import ( "fmt" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/utils" + "go.uber.org/zap" "os" "os/exec" "time" @@ -88,6 +89,7 @@ func (s *service) podmanRun(arg *podmanArgs) error { limit: utils.If(arg.limit == 0, 4*1024, arg.limit), timeout: utils.If(arg.timeout == 0, 10*time.Second, arg.timeout), kill: func() error { + s.log.Warn("[podman] timeout killer", zap.String("name", name)) if arg.kill != nil { _ = arg.kill() } diff --git a/pkg/file/writer.go b/pkg/file/writer.go index 797661a..ceaebd0 100644 --- a/pkg/file/writer.go +++ b/pkg/file/writer.go @@ -2,6 +2,7 @@ package file import ( "fmt" + "git.0x7f.app/WOJ/woj-server/pkg/utils" "os" ) @@ -11,11 +12,23 @@ type LimitedWriter struct { n int64 } -func (lw *LimitedWriter) Write(p []byte) (n int, err error) { - if lw.n+int64(len(p)) > lw.Limit { - return 0, fmt.Errorf("output limit exceeded") +func (lw *LimitedWriter) Write(p []byte) (int, error) { + remaining := lw.Limit - lw.n + if remaining <= 0 { + return 0, fmt.Errorf("write limit exceeded") } - n, err = lw.File.Write(p) + + var err, err2 error + var n int + + writeLen := int64(len(p)) + if writeLen > remaining { + err = fmt.Errorf("write limit exceeded") + writeLen = remaining + } + + n, err2 = lw.File.Write(p[:writeLen]) lw.n += int64(n) - return + + return n, utils.If(err2 != nil, err2, err) } From 802ce46f7f23eddb5c6cf880c9fdc5250a2aba97 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 7 Jan 2024 00:27:03 +0800 Subject: [PATCH 10/24] fix: ch1.3 case4 output is wrong --- .../runner/problem/book/ch1/3. acc_sum/data/output/4.output | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource/runner/problem/book/ch1/3. acc_sum/data/output/4.output b/resource/runner/problem/book/ch1/3. acc_sum/data/output/4.output index 1c288bb..a2c73af 100644 --- a/resource/runner/problem/book/ch1/3. acc_sum/data/output/4.output +++ b/resource/runner/problem/book/ch1/3. acc_sum/data/output/4.output @@ -1 +1 @@ -61382716 +395061382716 From 55f76636fd11e7f9d8e5b3fce2760d46627cefcb Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 7 Jan 2024 00:39:33 +0800 Subject: [PATCH 11/24] fix: asynq concurrency set to cpu num --- internal/app/runner/runner.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/app/runner/runner.go b/internal/app/runner/runner.go index 78cda77..4884196 100644 --- a/internal/app/runner/runner.go +++ b/internal/app/runner/runner.go @@ -6,10 +6,12 @@ import ( "git.0x7f.app/WOJ/woj-server/internal/misc/config" "git.0x7f.app/WOJ/woj-server/internal/misc/log" "git.0x7f.app/WOJ/woj-server/internal/model" + "git.0x7f.app/WOJ/woj-server/pkg/utils" "git.0x7f.app/WOJ/woj-server/pkg/zapasynq" "github.com/hibiken/asynq" "github.com/samber/do" "go.uber.org/zap" + "runtime" ) func RunRunner(i *do.Injector) error { @@ -32,7 +34,7 @@ func RunRunner(i *do.Injector) error { DB: conf.Redis.QueueDb, }, asynq.Config{ - Concurrency: 1, // there's a worker pool in runner service + Concurrency: utils.If(runtime.NumCPU() > 1, runtime.NumCPU()-1, 1), Logger: zapasynq.New(rlog), Queues: map[string]int{model.QueueRunner: 1}, }, From 7be0379e30125d0f0633f8ebb9385ede6cb07657 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 7 Jan 2024 00:39:48 +0800 Subject: [PATCH 12/24] chore: add exclude directory --- .idea/woj-server.iml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.idea/woj-server.iml b/.idea/woj-server.iml index 5e764c4..b91b798 100644 --- a/.idea/woj-server.iml +++ b/.idea/woj-server.iml @@ -2,7 +2,11 @@ - + + + + + From 7446ac6d23701c4cc2a5784e8afb99e51a4fbad4 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 7 Jan 2024 00:45:16 +0800 Subject: [PATCH 13/24] feat: let asynq retry submit request if it's internal error --- internal/api/runner/judge.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/api/runner/judge.go b/internal/api/runner/judge.go index 511d8a4..3716e9e 100644 --- a/internal/api/runner/judge.go +++ b/internal/api/runner/judge.go @@ -44,26 +44,31 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { if !h.runnerService.ProblemExists(p.ProblemVersionID) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) if status != e.Success { - return e.InternalError, 0, systemError + return status, 0, systemError } _, status = h.runnerService.NewProblem(p.ProblemVersionID, url, false) if status != e.Success { - return e.InternalError, 0, systemError + return status, 0, systemError } } // 3. compile compileResult, status := h.runnerService.Compile(p.ProblemVersionID, user, p.Submission.Language) if status != e.Success { - return e.InternalError, 0, compileResult + return status, 0, compileResult } // 4. run and judge result, point, status := h.runnerService.RunAndJudge(p.ProblemVersionID, user, p.Submission.Language) - return utils.If(status != e.Success, e.InternalError, e.Success), point, result + return status, point, result }() + if status == e.InternalError { + // notice asynq to retry + return fmt.Errorf("internal error, ctx: %v", ctx) + } + h.taskService.SubmitUpdate(&model.SubmitUpdatePayload{ Status: status, SubmissionID: p.Submission.ID, From b5bedddf94a3f443cf8f2760b2cb578a54e0fa7f Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 17:37:27 +0800 Subject: [PATCH 14/24] feat: use containerd as backend --- config.docker.yaml | 3 + docker-entrypoint.sh | 133 +++++++---- go.mod | 71 ++++-- go.sum | 221 +++++++++++++++--- internal/api/runner/build.go | 2 +- internal/api/runner/judge.go | 2 +- internal/model/config.go | 5 + internal/service/runner/common.go | 12 +- internal/service/runner/compile.go | 59 +++-- internal/service/runner/config.go | 4 +- internal/service/runner/container.go | 138 +++++++++++ internal/service/runner/deps.go | 74 +++--- internal/service/runner/exec.go | 111 --------- internal/service/runner/new_problem.go | 46 ++-- internal/service/runner/run_judge.go | 184 +++++++++------ internal/service/runner/service.go | 33 ++- resource/runner/scripts/common.sh | 7 +- resource/runner/scripts/ubuntu-run.Dockerfile | 6 +- 18 files changed, 745 insertions(+), 366 deletions(-) create mode 100644 internal/service/runner/container.go delete mode 100644 internal/service/runner/exec.go diff --git a/config.docker.yaml b/config.docker.yaml index 0e1ae51..283918f 100644 --- a/config.docker.yaml +++ b/config.docker.yaml @@ -37,6 +37,9 @@ Storage: SecretKey: ${STORAGE_SECRET_KEY} Bucket: ${STORAGE_BUCKET} +Runner: + Address: ${RUNNER_ADDRESS} + Metrics: Namespace: ${METRICS_NAMESPACE} Subsystem: ${METRICS_SUBSYSTEM} diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e05bb12..3f07562 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -9,8 +9,6 @@ function log_info() { echo -e "${COLOR_GREEN}$*${COLOR_NONE}" 1>&2; } function log_warn() { echo -e "${COLOR_YELLOW}$*${COLOR_NONE}" 1>&2; } function log_error() { echo -e "${COLOR_RED}$*${COLOR_NONE}" 1>&2; } -log_info "extracting env vars" - function check_env() { # $1 -> var name # $2 -> default value @@ -34,58 +32,105 @@ function check_env() { # echo " <<<<<" } -check_env "WEB_SERVER_ADDRESS" "0.0.0.0" true -check_env "WEB_SERVER_PORT" 8000 false -check_env "WEB_SERVER_PUBLIC_BASE" "http://127.0.0.1:8000" true -check_env "WEB_SERVER_TRUSTED_PLATFORM" "" true -check_env "WEB_SERVER_JWT_SIGNING_KEY" "$(head -n 10 /dev/urandom | md5sum | cut -c 1-32)" true -check_env "WEB_SERVER_JWT_EXPIRE_HOUR" 12 false -check_env "WEB_SERVER_OAUTH_DOMAIN" "" true -check_env "WEB_SERVER_OAUTH_CLIENT_ID" "" true -check_env "WEB_SERVER_OAUTH_CLIENT_SECRET" "" true +function extract_web_server() { + check_env "WEB_SERVER_ADDRESS" "0.0.0.0" true + check_env "WEB_SERVER_PORT" 8000 false + check_env "WEB_SERVER_PUBLIC_BASE" "http://127.0.0.1:8000" true + check_env "WEB_SERVER_TRUSTED_PLATFORM" "" true + check_env "WEB_SERVER_JWT_SIGNING_KEY" "$(head -n 10 /dev/urandom | md5sum | cut -c 1-32)" true + check_env "WEB_SERVER_JWT_EXPIRE_HOUR" 12 false + check_env "WEB_SERVER_OAUTH_DOMAIN" "" true + check_env "WEB_SERVER_OAUTH_CLIENT_ID" "" true + check_env "WEB_SERVER_OAUTH_CLIENT_SECRET" "" true +} -check_env "REDIS_DB" 0 false -check_env "REDIS_QUEUE_DB" 1 false -check_env "REDIS_ADDRESS" "redis" true -check_env "REDIS_PORT" 6379 false -check_env "REDIS_PASSWORD" "" true +function extract_redis() { + check_env "REDIS_DB" 0 false + check_env "REDIS_QUEUE_DB" 1 false + check_env "REDIS_ADDRESS" "redis" true + check_env "REDIS_PORT" 6379 false + check_env "REDIS_PASSWORD" "" true +} -check_env "DATABASE_HOST" "postgres" true -check_env "DATABASE_PORT" 5432 false -check_env "DATABASE_USER" "dev" true -check_env "DATABASE_PASSWORD" "password" true -check_env "DATABASE_NAME" "dev" true -check_env "DATABASE_PREFIX" "oj_" true -check_env "DATABASE_MAX_OPEN_CONNS" 100 false -check_env "DATABASE_MAX_IDLE_CONNS" 60 false -check_env "DATABASE_CONN_MAX_LIFETIME" 60 false -check_env "DATABASE_TIMEZONE" "Asia/Shanghai" true +function extract_database() { + check_env "DATABASE_HOST" "postgres" true + check_env "DATABASE_PORT" 5432 false + check_env "DATABASE_USER" "dev" true + check_env "DATABASE_PASSWORD" "password" true + check_env "DATABASE_NAME" "dev" true + check_env "DATABASE_PREFIX" "oj_" true + check_env "DATABASE_MAX_OPEN_CONNS" 100 false + check_env "DATABASE_MAX_IDLE_CONNS" 60 false + check_env "DATABASE_CONN_MAX_LIFETIME" 60 false + check_env "DATABASE_TIMEZONE" "Asia/Shanghai" true +} -check_env "STORAGE_ENDPOINT" "minio:9000" true -check_env "STORAGE_USE_SSL" "false" false -check_env "STORAGE_ACCESS_KEY" "access_key" true -check_env "STORAGE_SECRET_KEY" "secret_key" true -check_env "STORAGE_BUCKET" "woj" true +function extract_storage() { + check_env "STORAGE_ENDPOINT" "minio:9000" true + check_env "STORAGE_USE_SSL" "false" false + check_env "STORAGE_ACCESS_KEY" "access_key" true + check_env "STORAGE_SECRET_KEY" "secret_key" true + check_env "STORAGE_BUCKET" "woj" true +} -check_env "METRICS_NAMESPACE" "woj" true -check_env "METRICS_SUBSYSTEM" "server" true +function extract_runner() { + check_env "RUNNER_ADDRESS" "/run/containerd/containerd.sock" true +} -check_env "DEVELOPMENT" false false +function extract_metrics() { + check_env "METRICS_NAMESPACE" "woj" true + check_env "METRICS_SUBSYSTEM" "server" true +} -rm -f /tmp/tmp.yaml -( - echo "cat </app/config.yaml" - cat /app/config.docker.yaml - echo "EOF" -) >/tmp/tmp.yaml +function extract_misc() { + check_env "DEVELOPMENT" false false +} -if [ -f '/app/config.yaml' ]; then - log_info "config.yaml already exists, skip" -else +function generate_config() { + if [ -f '/app/config.yaml' ]; then + log_info "config.yaml already exists, skip" + return + fi + + # extract env vars + extract_web_server + extract_redis + extract_database + extract_storage + extract_runner + extract_metrics + extract_misc + + # dump script + rm -f /tmp/tmp.yaml + ( + echo "cat </app/config.yaml" + cat /app/config.docker.yaml + echo "EOF" + ) >/tmp/tmp.yaml + + # dump env vars log_info "creating config.yaml" . /tmp/tmp.yaml || (log_error "failed to create config.yaml" && exit 1) + + # cleanup + rm -f /tmp/tmp.yaml +} + +startup_containerd() { + rc-status + touch /run/openrc/softlevel + rc-update add containerd default + rc-service containerd start + rc-service containerd status + log_info 'containerd started' +} + +if [ -n "$START_CONTAINERD" ]; then + startup_containerd fi +generate_config + log_info "starting woj" -#cat /app/config.yaml exec /app/woj "$@" diff --git a/go.mod b/go.mod index b5becdf..df6ef7a 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,12 @@ module git.0x7f.app/WOJ/woj-server -go 1.21.5 +go 1.21.6 require ( github.com/TheZeroSlave/zapsentry v1.20.2 + github.com/containerd/containerd v1.7.12 github.com/coreos/go-oidc/v3 v3.9.0 - github.com/getsentry/sentry-go v0.25.0 + github.com/getsentry/sentry-go v0.26.0 github.com/gin-contrib/cors v1.5.0 github.com/gin-contrib/pprof v1.4.0 github.com/gin-contrib/zap v0.2.0 @@ -13,72 +14,97 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/hibiken/asynq v0.24.1 - github.com/jackc/pgtype v1.14.0 + github.com/jackc/pgtype v1.14.1 github.com/minio/minio-go/v7 v7.0.66 + github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4 github.com/prometheus/client_golang v1.18.0 - github.com/redis/go-redis/v9 v9.3.1 + github.com/redis/go-redis/v9 v9.4.0 github.com/samber/do v1.6.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.2 github.com/urfave/cli/v2 v2.27.1 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.17.0 - golang.org/x/oauth2 v0.15.0 + golang.org/x/crypto v0.18.0 + golang.org/x/oauth2 v0.16.0 golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.4 - gorm.io/gorm v1.25.5 + gorm.io/gorm v1.25.6 moul.io/zapgorm2 v1.3.0 ) require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.12.0-rc.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/fifo v1.1.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.2 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/spec v0.20.13 // indirect - github.com/go-openapi/swag v0.22.7 // indirect + github.com/go-openapi/spec v0.20.14 // indirect + github.com/go-openapi/swag v0.22.9 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.16.0 // indirect + github.com/go-playground/validator/v10 v10.17.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v4 v4.18.1 // indirect - github.com/jackc/pgx/v5 v5.5.1 // indirect + github.com/jackc/pgx/v5 v5.5.2 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.5 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/leodido/go-urn v1.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/moby/locker v1.0.1 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/signal v0.7.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc6 // indirect + github.com/opencontainers/selinux v1.11.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -88,14 +114,23 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/trace v1.22.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect + google.golang.org/grpc v1.61.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 160e843..8785fee 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,16 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.12.0-rc.2 h1:gfKebjq3Mq17Ys+4cjE8vc2h6tZVeqCGb9a7vBVqpAk= +github.com/Microsoft/hcsshim v0.12.0-rc.2/go.mod h1:G2TZhBED5frlh/hsuxV5CDh/ylkSFknPAMPpQg9owQw= github.com/TheZeroSlave/zapsentry v1.20.2 h1:llgC91ZJdoU/OzGxYpUlEhKinf65mw9hJ2KkZ7+cGIk= github.com/TheZeroSlave/zapsentry v1.20.2/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -17,6 +26,7 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -26,7 +36,23 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpV github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= +github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= +github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= +github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= +github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo= github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -40,14 +66,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= -github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.26.0 h1:IX3++sF6/4B5JcevhdZfdKIHfyvMmAq/UnqcyT2H6mA= +github.com/getsentry/sentry-go v0.26.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= @@ -69,14 +103,19 @@ github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkc github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkIaE= -github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= -github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= -github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= +github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= +github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= +github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= +github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -87,29 +126,53 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= -github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= +github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hibiken/asynq v0.24.1 h1:+5iIEAyA9K/lcSPvx3qoPtsKJeKI5u9aOIvUmSsazEw= github.com/hibiken/asynq v0.24.1/go.mod h1:u5qVeSbrnfT+vtG5Mq8ZPzQu/BmCKMHvTGb91uy9Tts= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= @@ -150,16 +213,17 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.1 h1:LyDar7M2K0tShCWqzJ/ctzF1QC3Wzc9c8a6cHE0PFdc= +github.com/jackc/pgtype v1.14.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= -github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI= -github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/pgx/v5 v5.5.2 h1:iLlpgp4Cp/gC9Xuscl7lFL1PhhW+ZLtXZcrfCt4C3tA= +github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -175,9 +239,10 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= +github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= @@ -196,8 +261,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ= +github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -213,19 +278,35 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= +github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4 h1:EctkgBjZ1y4q+sibyuuIgiKpa0QSd2elFtSSdNvBVow= +github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= @@ -239,15 +320,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= -github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= -github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -269,6 +352,7 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -288,7 +372,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -307,9 +390,21 @@ github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -348,33 +443,52 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -385,16 +499,20 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= @@ -417,28 +535,61 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe h1:USL2DhxfgRchafRvt/wYyyQNzwgL7ZiURcozOE/Pkvo= +google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= @@ -461,8 +612,10 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= -gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.6 h1:V92+vVda1wEISSOMtodHVRcUIOPYa2tgQtyF+DfFx+A= +gorm.io/gorm v1.25.6/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= diff --git a/internal/api/runner/build.go b/internal/api/runner/build.go index aaf76f3..5d4348b 100644 --- a/internal/api/runner/build.go +++ b/internal/api/runner/build.go @@ -17,7 +17,7 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry) } - h.log.Info("build", zap.Any("payload", p)) + h.log.Debug("build", zap.Any("payload", p)) status, ctx := func() (e.Status, string) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) diff --git a/internal/api/runner/judge.go b/internal/api/runner/judge.go index 3716e9e..83a2268 100644 --- a/internal/api/runner/judge.go +++ b/internal/api/runner/judge.go @@ -22,7 +22,7 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { } user := utils.RandomString(16) - h.log.Info("judge", zap.Any("payload", p), zap.String("user", user)) + h.log.Debug("judge", zap.Any("payload", p), zap.String("user", user)) status, point, ctx := func() (e.Status, int32, runner.JudgeStatus) { systemError := runner.JudgeStatus{ diff --git a/internal/model/config.go b/internal/model/config.go index b136ed6..d77fbe5 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -49,6 +49,10 @@ type ConfigStorage struct { Bucket string `yaml:"Bucket"` } +type ConfigRunner struct { + Address string `yaml:"Address"` +} + type ConfigMetrics struct { Namespace string `yaml:"Namespace"` Subsystem string `yaml:"Subsystem"` @@ -59,6 +63,7 @@ type Config struct { Redis ConfigRedis `yaml:"Redis"` Database ConfigDatabase `yaml:"Database"` Storage ConfigStorage `yaml:"Storage"` + Runner ConfigRunner `yaml:"Runner"` Metrics ConfigMetrics `yaml:"Metrics"` Development bool `yaml:"Development"` } diff --git a/internal/service/runner/common.go b/internal/service/runner/common.go index 9993d28..0f523ad 100644 --- a/internal/service/runner/common.go +++ b/internal/service/runner/common.go @@ -13,11 +13,15 @@ import ( var ( Prefix = "./resource/runner" ProblemDir = "./problem/" - ScriptsDir = "./scripts/" UserDir = "./user/" TmpDir = "./tmp/" ) +const ( + ContainerImageFull = "git.0x7f.app/woj/ubuntu-full:latest" + ContainerImageRun = "git.0x7f.app/woj/ubuntu-run:latest" +) + func init() { wd, err := os.Getwd() if err != nil { @@ -35,7 +39,7 @@ func (s *service) ProblemExists(version uint) bool { return file.Exist(problemPath) } -func (s *service) check(version uint, user string, lang string) e.Status { +func (s *service) ValidatePath(version uint, user string, lang string) e.Status { if !s.ProblemExists(version) { s.log.Info("problem not exists", zap.Uint("version", version)) return e.RunnerProblemNotExist @@ -50,7 +54,7 @@ func (s *service) check(version uint, user string, lang string) e.Status { return e.Success } -func (s *service) getLangInfo(config *Config, lang string) (configLanguage, bool) { +func (s *service) GetLangInfo(config *Config, lang string) (configLanguage, bool) { for _, l := range config.Languages { if l.Lang == lang { return l, true @@ -59,7 +63,7 @@ func (s *service) getLangInfo(config *Config, lang string) (configLanguage, bool return configLanguage{}, false } -func (s *service) getLangScript(l *configLanguage, lang string) string { +func (s *service) GetLangScript(l *configLanguage, lang string) string { if l.Type == "default" { return "/woj/framework/template/default/" + lang + ".Makefile" } else { diff --git a/internal/service/runner/compile.go b/internal/service/runner/compile.go index e4fca39..919c4ff 100644 --- a/internal/service/runner/compile.go +++ b/internal/service/runner/compile.go @@ -5,6 +5,7 @@ import ( "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/utils" + "github.com/opencontainers/runtime-spec/specs-go" "go.uber.org/zap" "io" "os" @@ -14,7 +15,7 @@ import ( func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, e.Status) { // 1. ensure problem/user exists - status := s.check(version, user, lang) + status := s.ValidatePath(version, user, lang) if status != e.Success { return JudgeStatus{Message: "check failed"}, status } @@ -53,32 +54,50 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, DoAny(func() error { return os.Remove(targetFile) }). Do(func() error { return file.TouchErr(targetFile) }). Do(func() error { - l, ok := s.getLangInfo(&config, lang) - script := s.getLangScript(&l, lang) + l, ok := s.GetLangInfo(&config, lang) + script := s.GetLangScript(&l, lang) if !ok { return e.RunnerProblemParseFailed.AsError() } - args := []string{ - "-v", fmt.Sprintf("%s:/woj/problem/judge:ro", judgeDir), - "-v", fmt.Sprintf("%s:/woj/user/%s.%s:ro", sourceFile, user, lang), - "-v", fmt.Sprintf("%s:/woj/user/%s.out", targetFile, user), - "-e", fmt.Sprintf("USER_PROG=%s", user), - "-e", fmt.Sprintf("LANG=%s", lang), - "git.0x7f.app/woj/ubuntu-full:latest", - "sh", "-c", fmt.Sprintf("cd /woj/user && make -f %s compile", script), - } - - runArgs := &podmanArgs{ - executeArgs: executeArgs{ - args: args, - timeout: 60 * time.Second, - output: log, + args := &RunArgs{ + Program: ProgramArgs{ + Args: []string{"sh", "-c", fmt.Sprintf("cd /woj/user && make -f %s compile", script)}, + Env: []string{fmt.Sprintf("USER_PROG=%s", user), fmt.Sprintf("LANG=%s", lang)}, + }, + Runtime: RuntimeArgs{ + Image: ContainerImageFull, + Memory: 256 * 1024 * 1024, // 256 MB + Timeout: time.Minute, + }, + IO: IOArgs{ + Output: log, + Limit: 4 * 1024, // 4 KB }, - memory: "256m", } - return s.podmanRun(runArgs) + args.Runtime.Mount = []specs.Mount{ + { + Source: judgeDir, + Destination: "/woj/problem/judge", + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: sourceFile, + Destination: fmt.Sprintf("/woj/user/%s.%s", user, lang), + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: targetFile, + Destination: fmt.Sprintf("/woj/user/%s.out", user), + Type: "bind", + Options: []string{"rbind"}, + }, + } + + return s.ContainerRun(args) }). Done() diff --git a/internal/service/runner/config.go b/internal/service/runner/config.go index 51fc1fa..d51ac3a 100644 --- a/internal/service/runner/config.go +++ b/internal/service/runner/config.go @@ -47,7 +47,7 @@ func (s *service) ParseConfig(version uint, skipCheck bool) (Config, error) { return config, nil } - err = s.checkConfig(&config, base) + err = s.ValidateConfig(&config, base) if err != nil { return Config{}, err } @@ -55,7 +55,7 @@ func (s *service) ParseConfig(version uint, skipCheck bool) (Config, error) { return config, nil } -func (s *service) checkConfig(config *Config, base string) error { +func (s *service) ValidateConfig(config *Config, base string) error { if config.Runtime.TimeLimit < 0 { return errors.New("time limit is negative") } diff --git a/internal/service/runner/container.go b/internal/service/runner/container.go new file mode 100644 index 0000000..3be2367 --- /dev/null +++ b/internal/service/runner/container.go @@ -0,0 +1,138 @@ +package runner + +import ( + "context" + "fmt" + "git.0x7f.app/WOJ/woj-server/pkg/file" + "git.0x7f.app/WOJ/woj-server/pkg/utils" + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "go.uber.org/zap" + "io" + "os" + "syscall" + "time" +) + +type ProgramArgs struct { + Args []string + Env []string +} + +type RuntimeArgs struct { + Image string + Pid int64 + Memory uint64 + Timeout time.Duration + Mount []specs.Mount +} + +func (r *RuntimeArgs) Normalize() { + r.Pid = utils.If(r.Pid <= 0, 64, r.Pid) + r.Memory = utils.If(r.Memory <= 0, 128*1024*1024, r.Memory) + r.Timeout = utils.If(r.Timeout <= 0, time.Minute, r.Timeout) +} + +type IOArgs struct { + Output *os.File + // Limit is the max size of output in chars. + // if Limit = 0, output to stderr if verbose, discard output if not. + // if Limit < 0, discard output. + Limit int64 +} + +type RunArgs struct { + Program ProgramArgs + Runtime RuntimeArgs + IO IOArgs +} + +func (s *service) ContainerRun(arg *RunArgs) error { + identifier := fmt.Sprintf("%d", s.container.count.Add(1)) + + // prepare args + arg.Runtime.Normalize() + + // prepare output + var writer io.Writer = nil + if arg.IO.Limit == 0 && s.verbose { + writer = os.Stderr + } else if arg.IO.Limit > 0 && arg.IO.Output != nil { + writer = &file.LimitedWriter{ + File: arg.IO.Output, + Limit: arg.IO.Limit, + } + } + + // debug log + s.log.Debug("container started", zap.String("identifier", identifier), zap.Any("args", arg)) + defer func(identifier string) { + s.log.Debug("container finished", zap.String("identifier", identifier)) + }(identifier) + + // get image + image, err := s.container.client.GetImage(s.container.ctx, arg.Runtime.Image) + // TODO: we could cache the image struct + if err != nil { + return err + } + + // create container + container, err := s.container.client.NewContainer(s.container.ctx, "task-"+identifier, + containerd.WithNewSnapshot("snapshot-"+identifier, image), + containerd.WithNewSpec( + oci.WithImageConfig(image), + oci.WithMemoryLimit(arg.Runtime.Memory), + oci.WithPidsLimit(arg.Runtime.Pid), + oci.WithMounts(arg.Runtime.Mount), + oci.WithProcessArgs(arg.Program.Args...), + oci.WithEnv(arg.Program.Env), + ), + ) + if err != nil { + return err + } + defer func(container containerd.Container, ctx context.Context, opts ...containerd.DeleteOpts) { + _ = container.Delete(ctx, opts...) + }(container, s.container.ctx, containerd.WithSnapshotCleanup) + + // create task + task, err := container.NewTask(s.container.ctx, cio.NewCreator(cio.WithStreams(nil, writer, writer))) + if err != nil { + return err + } + defer func(task containerd.Task, ctx context.Context, opts ...containerd.ProcessDeleteOpts) { + _, _ = task.Delete(ctx, opts...) + }(task, s.container.ctx, containerd.WithProcessKill) + + // wait + ctx2, cancel := context.WithTimeout(s.container.ctx, arg.Runtime.Timeout) + defer cancel() + exitStatusC, err := task.Wait(ctx2) + if err != nil { + return err + } + + // start + err = task.Start(s.container.ctx) + if err != nil { + return err + } + + // kill on timeout + status := <-exitStatusC + code, _, _ := status.Result() + if code == containerd.UnknownExitStatus { + // containerd is C/S architecture, timeout means grpc timeout, resulting in unknown exit status + // manually kill the task + s.log.Debug("container timeout", zap.String("identifier", identifier)) + err := task.Kill(s.container.ctx, syscall.SIGKILL) + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/service/runner/deps.go b/internal/service/runner/deps.go index 941b413..e8a913c 100644 --- a/internal/service/runner/deps.go +++ b/internal/service/runner/deps.go @@ -1,42 +1,56 @@ package runner import ( + "compress/gzip" "errors" - "fmt" "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/utils" + "github.com/containerd/containerd" "go.uber.org/zap" "os" "path/filepath" ) type depConfig struct { - tarball string - image string - dockerfile string + tarball string + image string } -func (s *service) loadImage(cfg *depConfig) e.Status { +func (s *service) LoadImageFromTarball(path string) error { + if !file.Exist(path) { + return errors.New("tarball not exists") + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + g, err := gzip.NewReader(f) + if err != nil { + return err + } + defer func(g *gzip.Reader) { + _ = g.Close() + }(g) + + _, err = s.container.client.Import(s.container.ctx, g) + return err +} + +func (s *service) LoadImageFromRegistry(image string) error { + _, err := s.container.client.Pull(s.container.ctx, image, containerd.WithPullUnpack) + return err +} + +func (s *service) LoadImage(cfg *depConfig) e.Status { err := utils.NewTryErr(). - Try(func() error { - // import from tarball - if !file.Exist(cfg.tarball) { - return errors.New("tarball not exists") - } - return s.execute("bash", "-c", fmt.Sprintf("gzip -d -c %s | podman load", cfg.tarball)) - }). - Or(func() error { - // pull from docker hub - return s.execute("podman", "pull", cfg.image) - }). - Or(func() error { - // build from dockerfile - if !file.Exist(cfg.dockerfile) { - return errors.New("dockerfile not exists") - } - return s.execute("podman", "build", "-f", cfg.dockerfile, "-t", cfg.image, ".") - }). + Try(func() error { return s.LoadImageFromTarball(cfg.tarball) }). + Or(func() error { return s.LoadImageFromRegistry(cfg.image) }). Done() if err != nil { @@ -59,21 +73,19 @@ func (s *service) EnsureDeps(force bool) e.Status { // full fullImage := &depConfig{ - tarball: filepath.Join(TmpDir, "ubuntu-full.tar.gz"), - image: "git.0x7f.app/woj/ubuntu-full:latest", - dockerfile: filepath.Join(ScriptsDir, "ubuntu-full.Dockerfile"), + tarball: filepath.Join(TmpDir, "ubuntu-full.tar.gz"), + image: ContainerImageFull, } - if s.loadImage(fullImage) != e.Success { + if s.LoadImage(fullImage) != e.Success { return e.RunnerDepsBuildFailed } // tiny tinyImage := &depConfig{ - tarball: filepath.Join(TmpDir, "ubuntu-tiny.tar.gz"), - image: "git.0x7f.app/woj/ubuntu-run:latest", - dockerfile: filepath.Join(ScriptsDir, "ubuntu-run.Dockerfile"), + tarball: filepath.Join(TmpDir, "ubuntu-tiny.tar.gz"), + image: ContainerImageRun, } - if s.loadImage(tinyImage) != e.Success { + if s.LoadImage(tinyImage) != e.Success { return e.RunnerDepsBuildFailed } diff --git a/internal/service/runner/exec.go b/internal/service/runner/exec.go deleted file mode 100644 index 79dd522..0000000 --- a/internal/service/runner/exec.go +++ /dev/null @@ -1,111 +0,0 @@ -package runner - -import ( - "context" - "errors" - "fmt" - "git.0x7f.app/WOJ/woj-server/pkg/file" - "git.0x7f.app/WOJ/woj-server/pkg/utils" - "go.uber.org/zap" - "os" - "os/exec" - "time" -) - -func (s *service) execute(exe string, args ...string) error { - cmd := exec.Command(exe, args...) - cmd.Dir = Prefix - if s.verbose { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } - return cmd.Run() -} - -type executeArgs struct { - exe string - args []string - - timeout time.Duration - kill func() error - - output *os.File - limit int64 -} - -func (s *service) executeTimeout(arg *executeArgs) error { - ctx, cancel := context.WithTimeout(context.Background(), arg.timeout) - defer cancel() - - cmd := exec.CommandContext(ctx, arg.exe, arg.args...) - cmd.Dir = Prefix - if arg.kill != nil { - cmd.Cancel = arg.kill - } - if s.verbose && arg.output == nil { - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - } else if arg.output != nil { - if arg.limit == 0 { - cmd.Stdout = arg.output - cmd.Stderr = arg.output - } else { - lw := &file.LimitedWriter{ - File: arg.output, - Limit: arg.limit, - } - cmd.Stdout = lw - cmd.Stderr = lw - } - } - - err := cmd.Start() - if err != nil { - return err - } - - err = cmd.Wait() - - // make sure the process is killed - _ = cmd.Process.Kill() - - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - return fmt.Errorf("command timed out") - } - - return err -} - -type podmanArgs struct { - executeArgs - memory string -} - -func (s *service) podmanRun(arg *podmanArgs) error { - name := fmt.Sprintf("woj-%d-%s", time.Now().UnixNano(), utils.RandomString(8)) - execArgs := &executeArgs{ - exe: "podman", - output: utils.If(arg.output == nil, os.Stderr, arg.output), - limit: utils.If(arg.limit == 0, 4*1024, arg.limit), - timeout: utils.If(arg.timeout == 0, 10*time.Second, arg.timeout), - kill: func() error { - s.log.Warn("[podman] timeout killer", zap.String("name", name)) - if arg.kill != nil { - _ = arg.kill() - } - return s.execute("podman", "kill", name) - }, - } - - args := []string{ - "run", - "--rm", - "--name", name, - "--memory", utils.If(arg.memory == "", "256m", arg.memory), - } - args = append(args, arg.args...) - - execArgs.args = args - - return s.executeTimeout(execArgs) -} diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index 2d89b77..687002e 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -6,13 +6,14 @@ import ( "git.0x7f.app/WOJ/woj-server/pkg/down" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/unzip" + "github.com/opencontainers/runtime-spec/specs-go" "go.uber.org/zap" "os" "path/filepath" "time" ) -func (s *service) download(version uint, url string) e.Status { +func (s *service) DownloadProblem(version uint, url string) e.Status { zipPath := filepath.Join(TmpDir, fmt.Sprintf("%d.zip", version)) problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) @@ -31,7 +32,7 @@ func (s *service) download(version uint, url string) e.Status { return e.Success } -func (s *service) prebuild(version uint, force bool) e.Status { +func (s *service) PrebuildProblem(version uint, force bool) e.Status { if !s.ProblemExists(version) { return e.RunnerProblemNotExist } @@ -52,21 +53,32 @@ func (s *service) prebuild(version uint, force bool) e.Status { dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") - args := []string{ - "-v", fmt.Sprintf("%s:/woj/problem/data", dataDir), - "-v", fmt.Sprintf("%s:/woj/problem/judge", judgeDir), - "-e", "PREFIX=/woj/problem", - "git.0x7f.app/woj/ubuntu-full:latest", - "sh", "-c", "cd /woj/problem/judge && make -f prebuild.Makefile prebuild && touch .mark.prebuild", - } - runArgs := &podmanArgs{ - executeArgs: executeArgs{ - args: args, - timeout: 300 * time.Second, + args := &RunArgs{ + Program: ProgramArgs{ + Args: []string{"sh", "-c", "cd /woj/problem/judge && make -f prebuild.Makefile prebuild && touch .mark.prebuild"}, + }, + Runtime: RuntimeArgs{ + Image: ContainerImageFull, + Memory: 256 * 1024 * 1024, // 256MB + Timeout: 5 * time.Minute, }, - memory: "1g", } - err := s.podmanRun(runArgs) + args.Runtime.Mount = []specs.Mount{ + { + Source: dataDir, + Destination: "/woj/problem/data", + Type: "bind", + Options: []string{"rbind"}, + }, + { + Source: judgeDir, + Destination: "/woj/problem/judge", + Type: "bind", + Options: []string{"rbind"}, + }, + } + + err := s.ContainerRun(args) if err != nil { s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", version)) @@ -83,7 +95,7 @@ func (s *service) NewProblem(version uint, url string, force bool) (Config, e.St } if !s.ProblemExists(version) { - status := s.download(version, url) + status := s.DownloadProblem(version, url) if status != e.Success { return Config{}, status } @@ -95,7 +107,7 @@ func (s *service) NewProblem(version uint, url string, force bool) (Config, e.St return Config{}, e.RunnerProblemParseFailed } - status := s.prebuild(version, true) + status := s.PrebuildProblem(version, true) if status != e.Success { return Config{}, status } diff --git a/internal/service/runner/run_judge.go b/internal/service/runner/run_judge.go index efbf9ea..c1cbf36 100644 --- a/internal/service/runner/run_judge.go +++ b/internal/service/runner/run_judge.go @@ -5,18 +5,38 @@ import ( "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/pkg/file" "git.0x7f.app/WOJ/woj-server/pkg/utils" + "github.com/opencontainers/runtime-spec/specs-go" "go.uber.org/zap" "os" "path/filepath" + "strings" "time" ) -func (s *service) problemRun(version uint, user string, lang string, config *Config) { +func (s *service) SandboxArgsBuilder(config *Config, user string, lang string, id int) string { + var args []string + + args = append(args, fmt.Sprintf("--memory_limit=%d", config.Runtime.MemoryLimit)) + args = append(args, fmt.Sprintf("--nproc_limit=%d", config.Runtime.NProcLimit)) + args = append(args, fmt.Sprintf("--time_limit=%d", config.Runtime.TimeLimit)) + args = append(args, fmt.Sprintf("--sandbox_template=%s", lang)) + args = append(args, fmt.Sprintf("--sandbox_action=ret")) + args = append(args, fmt.Sprintf("--uid=1000")) + args = append(args, fmt.Sprintf("--gid=1000")) + args = append(args, fmt.Sprintf("--file_input=/woj/problem/data/input/%d.input", id)) + args = append(args, fmt.Sprintf("--file_output=/woj/user/%d.out.usr", id)) + args = append(args, fmt.Sprintf("--file_info=/woj/user/%d.info", id)) + args = append(args, fmt.Sprintf("--program=/woj/user/%s.out", user)) + + return strings.Join(args, " ") +} + +func (s *service) ProblemRun(version uint, user string, lang string, config *Config) { workDir := filepath.Join(UserDir, user) dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data", "input") - // woj-sandbox killer will add 2 more seconds, here we add 2 more seconds - timeout := time.Duration((config.Runtime.TimeLimit+1000)/1000+2+2) * time.Second + // woj-sandbox killer will add 1 more second, here we add 1 also + timeout := time.Duration((config.Runtime.TimeLimit+1000)/1000+1+1) * time.Second ids := make([]int, 0) for _, task := range config.Tasks { @@ -27,51 +47,55 @@ func (s *service) problemRun(version uint, user string, lang string, config *Con ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) ifoFile := filepath.Join(workDir, fmt.Sprintf("%d.info", id)) + args := &RunArgs{ + Program: ProgramArgs{ + Args: []string{ + "sh", "-c", + "cd /woj/user && /woj/framework/scripts/woj_launcher " + + s.SandboxArgsBuilder(config, user, lang, id), + }, + }, + Runtime: RuntimeArgs{ + Image: ContainerImageRun, + // sh, woj_launcher:program, woj_launcher:killer, woj_launcher:stat + Pid: int64(config.Runtime.NProcLimit + 4), + Memory: uint64(config.Runtime.MemoryLimit * 1024 * 1024), + Timeout: timeout, + }, + } + args.Runtime.Mount = []specs.Mount{ + { + Source: testCase, + Destination: fmt.Sprintf("/woj/problem/data/input/%d.input", id), + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: targetFile, + Destination: fmt.Sprintf("/woj/user/%s.out", user), + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: ansFile, + Destination: fmt.Sprintf("/woj/user/%d.out.usr", id), + Type: "bind", + Options: []string{"rbind"}, + }, + { + Source: ifoFile, + Destination: fmt.Sprintf("/woj/user/%d.info", id), + Type: "bind", + Options: []string{"rbind"}, + }, + } + err := utils.NewMust(). DoAny(func() error { return os.Remove(ansFile) }). DoAny(func() error { return os.Remove(ifoFile) }). Do(func() error { return file.TouchErr(ansFile) }). Do(func() error { return file.TouchErr(ifoFile) }). - Do(func() error { - args := []string{ - "--cpus", "1", - "--network", "none", - "-v", fmt.Sprintf("%s:/woj/problem/data/input/%d.input:ro", testCase, id), - "-v", fmt.Sprintf("%s:/woj/user/%s.out:ro", targetFile, user), - "-v", fmt.Sprintf("%s:/woj/user/%d.out.usr", ansFile, id), - "-v", fmt.Sprintf("%s:/woj/user/%d.info", ifoFile, id), - "git.0x7f.app/woj/ubuntu-run:latest", - "sh", "-c", - fmt.Sprintf("cd /woj/user && /woj/framework/scripts/woj_launcher "+ - "--memory_limit=%d "+ - "--nproc_limit=%d "+ - "--time_limit=%d "+ - "--sandbox_template=%s "+ - "--sandbox_action=ret "+ - "--uid=1000 "+ - "--gid=1000 "+ - "--file_input=/woj/problem/data/input/%d.input "+ - "--file_output=/woj/user/%d.out.usr "+ - "--file_info=/woj/user/%d.info "+ - "-program=/woj/user/%s.out", - config.Runtime.MemoryLimit, - config.Runtime.NProcLimit, - config.Runtime.TimeLimit, - lang, - id, id, id, - user, - ), - } - - runArgs := &podmanArgs{ - executeArgs: executeArgs{ - args: args, - timeout: timeout, - }, - } - - return s.podmanRun(runArgs) - }). + Do(func() error { return s.ContainerRun(args) }). Done() if err != nil { @@ -94,7 +118,7 @@ func (s *service) problemRun(version uint, user string, lang string, config *Con } } -func (s *service) problemJudge(version uint, user string, lang string, config *Config) { +func (s *service) ProblemJudge(version uint, user string, lang string, config *Config) { workDir := filepath.Join(UserDir, user) dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") @@ -106,35 +130,57 @@ func (s *service) problemJudge(version uint, user string, lang string, config *C ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) jdgFile := filepath.Join(workDir, fmt.Sprintf("%d.judge", id)) - c, ok := s.getLangInfo(config, lang) + c, ok := s.GetLangInfo(config, lang) if !ok { return } + args := &RunArgs{ + Program: ProgramArgs{ + Args: []string{ + "sh", "-c", + fmt.Sprintf("cd /woj/user && make -f %s judge", s.GetLangScript(&c, lang)), + }, + Env: []string{ + fmt.Sprintf("TEST_NUM=%d", id), + fmt.Sprintf("CMP=%s", c.Cmp), + }, + }, + Runtime: RuntimeArgs{ + Image: ContainerImageFull, + }, + } + args.Runtime.Mount = []specs.Mount{ + { + Source: judgeDir, + Destination: "/woj/problem/judge", + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: dataDir, + Destination: "/woj/problem/data", + Type: "bind", + Options: []string{"rbind", "ro"}, + }, + { + Source: ansFile, + Destination: fmt.Sprintf("/woj/user/%d.out.usr", id), + Type: "bind", + Options: []string{"rbind"}, + }, + { + Source: jdgFile, + Destination: fmt.Sprintf("/woj/user/%d.judge", id), + Type: "bind", + Options: []string{"rbind"}, + }, + } + err := utils.NewMust(). DoAny(func() error { return os.Remove(jdgFile) }). Do(func() error { return file.TouchErr(jdgFile) }). - Do(func() error { - args := []string{ - "-v", fmt.Sprintf("%s:/woj/problem/judge:ro", judgeDir), - "-v", fmt.Sprintf("%s:/woj/problem/data:ro", dataDir), - "-v", fmt.Sprintf("%s:/woj/user/%d.out.usr", ansFile, id), - "-v", fmt.Sprintf("%s:/woj/user/%d.judge", jdgFile, id), - "-e", fmt.Sprintf("TEST_NUM=%d", id), - "-e", fmt.Sprintf("CMP=%s", c.Cmp), - "git.0x7f.app/woj/ubuntu-full:latest", - "sh", "-c", - fmt.Sprintf("cd /woj/user && make -f %s judge", s.getLangScript(&c, lang)), - } - - runArgs := &podmanArgs{ - executeArgs: executeArgs{ - args: args, - }, - } - - return s.podmanRun(runArgs) - }). + Do(func() error { return s.ContainerRun(args) }). Done() if err != nil { @@ -159,7 +205,7 @@ func (s *service) problemJudge(version uint, user string, lang string, config *C func (s *service) RunAndJudge(version uint, user string, lang string) (JudgeStatus, int32, e.Status) { // 1. ensure problem/user exists - status := s.check(version, user, lang) + status := s.ValidatePath(version, user, lang) if status != e.Success { return JudgeStatus{Message: "check failed"}, 0, status } @@ -171,10 +217,10 @@ func (s *service) RunAndJudge(version uint, user string, lang string) (JudgeStat } // 3. run user program - s.problemRun(version, user, lang, &config) + s.ProblemRun(version, user, lang, &config) // 4. run judger - s.problemJudge(version, user, lang, &config) + s.ProblemJudge(version, user, lang, &config) // 5. check result result, pts := s.checkResults(user, &config) diff --git a/internal/service/runner/service.go b/internal/service/runner/service.go index 6c3c97b..ab5c826 100644 --- a/internal/service/runner/service.go +++ b/internal/service/runner/service.go @@ -1,14 +1,18 @@ package runner import ( + "context" "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/internal/misc/config" "git.0x7f.app/WOJ/woj-server/internal/misc/log" "git.0x7f.app/WOJ/woj-server/pkg/pool" "git.0x7f.app/WOJ/woj-server/pkg/utils" + "github.com/containerd/containerd" + "github.com/containerd/containerd/namespaces" "github.com/samber/do" "go.uber.org/zap" "runtime" + "sync/atomic" ) var _ Service = (*service)(nil) @@ -35,20 +39,36 @@ type Service interface { func NewService(i *do.Injector) (Service, error) { concurrency := utils.If(runtime.NumCPU() > 1, runtime.NumCPU()-1, 1) + cfg := do.MustInvoke[config.Service](i).GetConfig() srv := &service{ log: do.MustInvoke[log.Service](i).GetLogger("runner"), pool: pool.NewTaskPool(concurrency, concurrency), - verbose: do.MustInvoke[config.Service](i).GetConfig().Development, + verbose: cfg.Development, } - srv.pool.Start() + var err error + srv.container.client, err = containerd.New(cfg.Runner.Address) + if err != nil { + srv.log.Error("failed to connect to containerd", zap.Error(err)) + return nil, err + } + + srv.container.ctx = namespaces.WithNamespace(context.Background(), "woj") + srv.container.count.Store(0) + + srv.pool.Start() return srv, nil } type service struct { - log *zap.Logger - pool *pool.TaskPool + log *zap.Logger + pool *pool.TaskPool + container struct { + client *containerd.Client + ctx context.Context + count atomic.Uint64 + } verbose bool } @@ -58,5 +78,10 @@ func (s *service) HealthCheck() error { func (s *service) Shutdown() error { s.pool.Stop() + if s.container.client != nil { + // TODO: wait and kill all containers + _ = s.container.client.Close() + } + return nil } diff --git a/resource/runner/scripts/common.sh b/resource/runner/scripts/common.sh index 408ed96..76a2ef5 100755 --- a/resource/runner/scripts/common.sh +++ b/resource/runner/scripts/common.sh @@ -9,9 +9,6 @@ function log_info() { echo -e "${COLOR_GREEN}$*${COLOR_NONE}" 1>&2; } function log_warn() { echo -e "${COLOR_YELLOW}$*${COLOR_NONE}" 1>&2; } function log_error() { echo -e "${COLOR_RED}$*${COLOR_NONE}" 1>&2; } -# Docker or Podman -DOCKER="podman" -if [ "$USE_DOCKER" ]; then - log_error "docker is deprecated" - log_info "Use podman instead" +if [ -z "$DOCKER" ]; then + DOCKER="nerdctl" fi diff --git a/resource/runner/scripts/ubuntu-run.Dockerfile b/resource/runner/scripts/ubuntu-run.Dockerfile index 9221fec..aefef83 100644 --- a/resource/runner/scripts/ubuntu-run.Dockerfile +++ b/resource/runner/scripts/ubuntu-run.Dockerfile @@ -1,11 +1,7 @@ -FROM woj/ubuntu-full:latest AS builder +FROM git.0x7f.app/woj/ubuntu-full:latest AS builder FROM docker.io/library/ubuntu:22.04 WORKDIR /woj RUN mkdir -p /woj/framework/scripts COPY --from=builder /woj/framework/scripts/woj_launcher /woj/framework/scripts/ - -# Add User -RUN groupadd -g 1000 woj && useradd -M -u 1000 -g 1000 woj - From 362b5ea9bab993f080fd64a616cbb7da8d20e788 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 19:11:26 +0800 Subject: [PATCH 15/24] chore: update docker scripts --- Runner.Dockerfile | 16 ++++------------ Server.Dockerfile | 4 ++-- build_image.sh | 3 +-- docker-compose.yml | 14 +++++--------- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/Runner.Dockerfile b/Runner.Dockerfile index d592aef..385dd15 100644 --- a/Runner.Dockerfile +++ b/Runner.Dockerfile @@ -17,24 +17,16 @@ RUN --mount=type=cache,id=golang,target=/go/pkg make build # main image -FROM quay.io/podman/stable - -# pkill -RUN yum -y install jq procps-ng && yum -y clean all && rm -rf /var/cache +FROM docker.io/library/alpine WORKDIR /app - -# prepare images -COPY --from=builder /builder/resource/runner /app/resource/runner -RUN bash -c "cd /app/resource/runner/scripts && ./prepare_images.sh save" +RUN apk --no-cache add tzdata ca-certificates bash openrc \ + containerd nerdctl # sources +COPY --from=builder /builder/resource/runner /app/resource/runner COPY --from=builder /builder/config.docker.yaml /app COPY --from=builder /builder/docker-entrypoint.sh /app COPY --from=builder /builder/woj /app -# switch user -RUN chown -R podman:podman /app -USER podman - ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/Server.Dockerfile b/Server.Dockerfile index f4625e5..ac36edc 100644 --- a/Server.Dockerfile +++ b/Server.Dockerfile @@ -1,7 +1,7 @@ # Go builder FROM docker.io/library/golang:alpine AS go-builder -ENV GOPROXY=https://goproxy.cn +#ENV GOPROXY=https://goproxy.cn ENV CGO_ENABLED=0 WORKDIR /builder @@ -23,7 +23,7 @@ RUN find /app -type f -name "*.map" -delete FROM docker.io/library/alpine WORKDIR /app -RUN apk --no-cache add tzdata ca-certificates libc6-compat bash +RUN apk --no-cache add tzdata ca-certificates bash COPY --from=go-builder /builder/config.docker.yaml /app COPY --from=go-builder /builder/docker-entrypoint.sh /app diff --git a/build_image.sh b/build_image.sh index c85bb03..97d5b59 100755 --- a/build_image.sh +++ b/build_image.sh @@ -31,8 +31,7 @@ function build_server() { function build_runner() { log_info "[+] Building Runner" $DOCKER build \ - --cap-add=sys_admin,mknod \ - --device=/dev/fuse \ + --cap-add=sys_admin \ --security-opt label=disable \ -t "git.0x7f.app/woj/woj-runner:latest" \ -f Runner.Dockerfile . || diff --git a/docker-compose.yml b/docker-compose.yml index 3c412e3..519e792 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: server: - image: git.0x7f.app/woj/woj-server:1.2.2 + image: git.0x7f.app/woj/woj-server:1.2.3-dev restart: unless-stopped healthcheck: test: [ "CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8000/health" ] @@ -21,8 +21,6 @@ services: - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro depends_on: - runner: - condition: service_started storage: condition: service_healthy cache: @@ -33,16 +31,12 @@ services: - "8000:8000" runner: - image: git.0x7f.app/woj/woj-runner:1.2.2 + image: git.0x7f.app/woj/woj-runner:1.2.3-dev restart: unless-stopped command: runner - security_opt: - - "label=disable" + privileged: true cap_add: - SYS_ADMIN - - MKNOD - devices: - - "/dev/fuse" environment: - REDIS_ADDRESS=cache - STORAGE_ENDPOINT=storage:9000 @@ -50,8 +44,10 @@ services: - STORAGE_SECRET_KEY=secret_key - STORAGE_BUCKET=woj - DEVELOPMENT=true + - START_CONTAINERD=true volumes: - runner:/app/resource/runner/user + - container:/var/lib/containerd - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro depends_on: From 025c88ca6b58483e36bec4842164d7f614ea1de1 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 19:54:09 +0800 Subject: [PATCH 16/24] fix: drop openrc in runner images: missing pids in cgroups setup --- Runner.Dockerfile | 5 +++-- docker-entrypoint.sh | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Runner.Dockerfile b/Runner.Dockerfile index 385dd15..2072f2a 100644 --- a/Runner.Dockerfile +++ b/Runner.Dockerfile @@ -20,7 +20,7 @@ RUN --mount=type=cache,id=golang,target=/go/pkg make build FROM docker.io/library/alpine WORKDIR /app -RUN apk --no-cache add tzdata ca-certificates bash openrc \ +RUN apk --no-cache add tzdata ca-certificates bash tini \ containerd nerdctl # sources @@ -29,4 +29,5 @@ COPY --from=builder /builder/config.docker.yaml /app COPY --from=builder /builder/docker-entrypoint.sh /app COPY --from=builder /builder/woj /app -ENTRYPOINT ["/app/docker-entrypoint.sh"] +# reap zombies from containerd-shim +ENTRYPOINT ["/sbin/tini", "/app/docker-entrypoint.sh"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 3f07562..df7a694 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -118,11 +118,19 @@ function generate_config() { } startup_containerd() { - rc-status - touch /run/openrc/softlevel - rc-update add containerd default - rc-service containerd start - rc-service containerd status + # taken from https://github.com/moby/moby/blob/ee6cbc540e9c62feb143c2a8d3f0c86d2a468767/hack/dind#L59-L69 + # cgroup v2: enable nesting + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + # move the processes from the root group to the /init group, + # otherwise writing subtree_control fails with EBUSY. + # An error during moving non-existent process (i.e., "cat") is ignored. + mkdir -p /sys/fs/cgroup/init + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + # enable controllers + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control + fi + nohup containerd & log_info 'containerd started' } From bda209f794e0b3400f9ca174a031ba4456313c1b Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 21:59:38 +0800 Subject: [PATCH 17/24] chore: upgrade problem and ubuntu-full to next version --- .idea/codeStyles/codeStyleConfig.xml | 5 ++ resource/runner/problem/.gitignore | 2 +- .../problem/book/ch1/1. factorial/config.json | 60 +++++-------- .../problem/book/ch1/2. divide/config.json | 65 +++++--------- .../problem/book/ch1/3. acc_sum/config.json | 60 +++++-------- .../problem/book/ch1/4. f_sum/config.json | 85 ++++++------------- .../problem/book/ch1/5. multiply/config.json | 85 ++++++------------- .../problem/book/ch1/6. yubikili/config.json | 60 +++++-------- .../problem/book/ch1/7. hanoi/config.json | 85 ++++++------------- .../problem/book/ch2/1. random/config.json | 47 +++++----- .../problem/book/ch2/2. carry/config.json | 47 +++++----- .../problem/book/ch2/3. masses/config.json | 47 +++++----- .../problem/book/ch2/4. knumber/config.json | 47 +++++----- .../problem/book/ch2/5. secret/config.json | 47 +++++----- .../problem/book/ch2/6. scholar/config.json | 47 +++++----- .../problem/book/ch2/7. count/config.json | 47 +++++----- .../problem/book/ch2/8. pipe/config.json | 47 +++++----- .../problem/book/ch2/9. sol/config.json | 47 +++++----- .../problem/book/ch3/1. stairs/config.json | 47 +++++----- .../problem/book/ch3/2. rabbit/config.json | 47 +++++----- .../problem/book/ch3/3. surface/config.json | 47 +++++----- .../problem/book/ch3/4. domino/config.json | 47 +++++----- .../problem/book/ch3/5. bee/config.json | 37 ++++---- .../problem/book/ch3/6. acme/config.json | 37 ++++---- .../problem/book/ch3/7. railway/config.json | 37 ++++---- resource/runner/problem/example/README.md | 48 +++++++---- resource/runner/problem/example/config.json | 26 ++++-- .../judge/{XYZ.Makefile => custom.Makefile} | 3 - .../runner/scripts/ubuntu-full.Dockerfile | 14 ++- 29 files changed, 602 insertions(+), 718 deletions(-) create mode 100644 .idea/codeStyles/codeStyleConfig.xml rename resource/runner/problem/example/judge/{XYZ.Makefile => custom.Makefile} (95%) diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/resource/runner/problem/.gitignore b/resource/runner/problem/.gitignore index a82daa9..136b83a 100644 --- a/resource/runner/problem/.gitignore +++ b/resource/runner/problem/.gitignore @@ -1,6 +1,6 @@ * !.gitignore !example -!example/* +!example/**/* !book !book/**/* diff --git a/resource/runner/problem/book/ch1/1. factorial/config.json b/resource/runner/problem/book/ch1/1. factorial/config.json index 2bc2544..7043a63 100644 --- a/resource/runner/problem/book/ch1/1. factorial/config.json +++ b/resource/runner/problem/book/ch1/1. factorial/config.json @@ -1,43 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 20 - }, - { - "Id": 2, - "Points": 20 - }, - { - "Id": 3, - "Points": 20 - }, - { - "Id": 4, - "Points": 20 - }, - { - "Id": 5, - "Points": 20 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/2. divide/config.json b/resource/runner/problem/book/ch1/2. divide/config.json index 264d4a4..f310200 100644 --- a/resource/runner/problem/book/ch1/2. divide/config.json +++ b/resource/runner/problem/book/ch1/2. divide/config.json @@ -1,47 +1,22 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "FCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "FCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 10 - }, - { - "Id": 2, - "Points": 10 - }, - { - "Id": 3, - "Points": 10 - }, - { - "Id": 4, - "Points": 23 - }, - { - "Id": 5, - "Points": 22 - }, - { - "Id": 6, - "Points": 25 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "FCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "FCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 23}, + {"Id": 5, "Points": 22}, + {"Id": 6, "Points": 25} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/3. acc_sum/config.json b/resource/runner/problem/book/ch1/3. acc_sum/config.json index 808439f..55f82ef 100644 --- a/resource/runner/problem/book/ch1/3. acc_sum/config.json +++ b/resource/runner/problem/book/ch1/3. acc_sum/config.json @@ -1,43 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "NCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "NCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 20 - }, - { - "Id": 2, - "Points": 20 - }, - { - "Id": 3, - "Points": 20 - }, - { - "Id": 4, - "Points": 20 - }, - { - "Id": 5, - "Points": 20 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/4. f_sum/config.json b/resource/runner/problem/book/ch1/4. f_sum/config.json index f5b3106..285cf43 100644 --- a/resource/runner/problem/book/ch1/4. f_sum/config.json +++ b/resource/runner/problem/book/ch1/4. f_sum/config.json @@ -1,63 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 10 - }, - { - "Id": 2, - "Points": 10 - }, - { - "Id": 3, - "Points": 10 - }, - { - "Id": 4, - "Points": 10 - }, - { - "Id": 5, - "Points": 10 - }, - { - "Id": 6, - "Points": 10 - }, - { - "Id": 7, - "Points": 10 - }, - { - "Id": 8, - "Points": 10 - }, - { - "Id": 9, - "Points": 10 - }, - { - "Id": 10, - "Points": 10 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/5. multiply/config.json b/resource/runner/problem/book/ch1/5. multiply/config.json index f5b3106..285cf43 100644 --- a/resource/runner/problem/book/ch1/5. multiply/config.json +++ b/resource/runner/problem/book/ch1/5. multiply/config.json @@ -1,63 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 10 - }, - { - "Id": 2, - "Points": 10 - }, - { - "Id": 3, - "Points": 10 - }, - { - "Id": 4, - "Points": 10 - }, - { - "Id": 5, - "Points": 10 - }, - { - "Id": 6, - "Points": 10 - }, - { - "Id": 7, - "Points": 10 - }, - { - "Id": 8, - "Points": 10 - }, - { - "Id": 9, - "Points": 10 - }, - { - "Id": 10, - "Points": 10 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/6. yubikili/config.json b/resource/runner/problem/book/ch1/6. yubikili/config.json index 2bc2544..7043a63 100644 --- a/resource/runner/problem/book/ch1/6. yubikili/config.json +++ b/resource/runner/problem/book/ch1/6. yubikili/config.json @@ -1,43 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 20 - }, - { - "Id": 2, - "Points": 20 - }, - { - "Id": 3, - "Points": 20 - }, - { - "Id": 4, - "Points": 20 - }, - { - "Id": 5, - "Points": 20 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch1/7. hanoi/config.json b/resource/runner/problem/book/ch1/7. hanoi/config.json index f5b3106..285cf43 100644 --- a/resource/runner/problem/book/ch1/7. hanoi/config.json +++ b/resource/runner/problem/book/ch1/7. hanoi/config.json @@ -1,63 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { - "Lang": "c", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - }, - { - "Lang": "cpp", - "Type": "default", - "Script": "", - "Cmp": "HCMP" - } - ], - "Tasks": [ - { - "Id": 1, - "Points": 10 - }, - { - "Id": 2, - "Points": 10 - }, - { - "Id": 3, - "Points": 10 - }, - { - "Id": 4, - "Points": 10 - }, - { - "Id": 5, - "Points": 10 - }, - { - "Id": 6, - "Points": 10 - }, - { - "Id": 7, - "Points": 10 - }, - { - "Id": 8, - "Points": 10 - }, - { - "Id": 9, - "Points": 10 - }, - { - "Id": 10, - "Points": 10 - } - ] + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "HCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] } \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/1. random/config.json b/resource/runner/problem/book/ch2/1. random/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/1. random/config.json +++ b/resource/runner/problem/book/ch2/1. random/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/2. carry/config.json b/resource/runner/problem/book/ch2/2. carry/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/2. carry/config.json +++ b/resource/runner/problem/book/ch2/2. carry/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/3. masses/config.json b/resource/runner/problem/book/ch2/3. masses/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/3. masses/config.json +++ b/resource/runner/problem/book/ch2/3. masses/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/4. knumber/config.json b/resource/runner/problem/book/ch2/4. knumber/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/4. knumber/config.json +++ b/resource/runner/problem/book/ch2/4. knumber/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/5. secret/config.json b/resource/runner/problem/book/ch2/5. secret/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/5. secret/config.json +++ b/resource/runner/problem/book/ch2/5. secret/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/6. scholar/config.json b/resource/runner/problem/book/ch2/6. scholar/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/6. scholar/config.json +++ b/resource/runner/problem/book/ch2/6. scholar/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/7. count/config.json b/resource/runner/problem/book/ch2/7. count/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/7. count/config.json +++ b/resource/runner/problem/book/ch2/7. count/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/8. pipe/config.json b/resource/runner/problem/book/ch2/8. pipe/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/8. pipe/config.json +++ b/resource/runner/problem/book/ch2/8. pipe/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch2/9. sol/config.json b/resource/runner/problem/book/ch2/9. sol/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch2/9. sol/config.json +++ b/resource/runner/problem/book/ch2/9. sol/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/1. stairs/config.json b/resource/runner/problem/book/ch3/1. stairs/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch3/1. stairs/config.json +++ b/resource/runner/problem/book/ch3/1. stairs/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/2. rabbit/config.json b/resource/runner/problem/book/ch3/2. rabbit/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch3/2. rabbit/config.json +++ b/resource/runner/problem/book/ch3/2. rabbit/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/3. surface/config.json b/resource/runner/problem/book/ch3/3. surface/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch3/3. surface/config.json +++ b/resource/runner/problem/book/ch3/3. surface/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/4. domino/config.json b/resource/runner/problem/book/ch3/4. domino/config.json index 45b2d86..e5deb2b 100644 --- a/resource/runner/problem/book/ch3/4. domino/config.json +++ b/resource/runner/problem/book/ch3/4. domino/config.json @@ -1,23 +1,26 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 10 }, - { "Id": 2, "Points": 10 }, - { "Id": 3, "Points": 10 }, - { "Id": 4, "Points": 10 }, - { "Id": 5, "Points": 10 }, - { "Id": 6, "Points": 10 }, - { "Id": 7, "Points": 10 }, - { "Id": 8, "Points": 10 }, - { "Id": 9, "Points": 10 }, - { "Id": 10, "Points": 10 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 10}, + {"Id": 2, "Points": 10}, + {"Id": 3, "Points": 10}, + {"Id": 4, "Points": 10}, + {"Id": 5, "Points": 10}, + {"Id": 6, "Points": 10}, + {"Id": 7, "Points": 10}, + {"Id": 8, "Points": 10}, + {"Id": 9, "Points": 10}, + {"Id": 10, "Points": 10} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/5. bee/config.json b/resource/runner/problem/book/ch3/5. bee/config.json index a36b8a9..55f82ef 100644 --- a/resource/runner/problem/book/ch3/5. bee/config.json +++ b/resource/runner/problem/book/ch3/5. bee/config.json @@ -1,18 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 20 }, - { "Id": 2, "Points": 20 }, - { "Id": 3, "Points": 20 }, - { "Id": 4, "Points": 20 }, - { "Id": 5, "Points": 20 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/6. acme/config.json b/resource/runner/problem/book/ch3/6. acme/config.json index 378c6d6..8d3128b 100644 --- a/resource/runner/problem/book/ch3/6. acme/config.json +++ b/resource/runner/problem/book/ch3/6. acme/config.json @@ -1,18 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "LCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "LCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 20 }, - { "Id": 2, "Points": 20 }, - { "Id": 3, "Points": 20 }, - { "Id": 4, "Points": 20 }, - { "Id": 5, "Points": 20 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "LCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "LCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/book/ch3/7. railway/config.json b/resource/runner/problem/book/ch3/7. railway/config.json index a36b8a9..55f82ef 100644 --- a/resource/runner/problem/book/ch3/7. railway/config.json +++ b/resource/runner/problem/book/ch3/7. railway/config.json @@ -1,18 +1,21 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, - "Languages": [ - { "Lang": "c", "Type": "default", "Script": "", "Cmp": "NCMP" }, - { "Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP" } - ], - "Tasks": [ - { "Id": 1, "Points": 20 }, - { "Id": 2, "Points": 20 }, - { "Id": 3, "Points": 20 }, - { "Id": 4, "Points": 20 }, - { "Id": 5, "Points": 20 } - ] -} + "Languages": [ + { + "Lang" : "c", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + }, + { + "Lang" : "cpp", + "Judge:" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": {"Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}} + } + ], + "Tasks" : [ + {"Id": 1, "Points": 20}, + {"Id": 2, "Points": 20}, + {"Id": 3, "Points": 20}, + {"Id": 4, "Points": 20}, + {"Id": 5, "Points": 20} + ] +} \ No newline at end of file diff --git a/resource/runner/problem/example/README.md b/resource/runner/problem/example/README.md index d68454a..14f9f0d 100644 --- a/resource/runner/problem/example/README.md +++ b/resource/runner/problem/example/README.md @@ -25,22 +25,40 @@ ```json5 { - "Runtime": { - // 运行时配置 - "TimeLimit": 1000, // 时间限制 (ms) - "MemoryLimit": 16, // 内存限制 (MB) - "NProcLimit": 1 // 进(线)程 限制 - }, - "Languages": [ // 支持的语言 - // c 语言，使用自定义评测脚本，脚本为 ./judge/XYZ.Makefile - {"Lang": "c", "Type": "custom", "Script": "XYZ.Makefile", "Cmp": ""}, - // c++ 语言，使用默认评测脚本，答案比对方式为 NCMP(testlib) - {"Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP"} - ], - "Tasks": [ - // 评测点信息 - {"Id": 1, "Points": 10}, // 第一个评测点，分值 25 分，使用 ./data/{input,output}/1.{input,output} 为测试数据 + "Languages": [ + { + // C 语言 + "Lang" : "c", + // 使用自定义评测脚本，脚本为 ./judge/custom.Makefile，Cmp 将被忽略 + "Judge" : {"Type": "custom", "Script": "custom.Makefile", "Cmp": ""}, + // 运行时配置：时间(ms) 内存(MB) 进/线程数目 + "Runtime": { + // 题目构建阶段，用于生成测试数据等，可选，默认值见下 + "Prebuild": {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, + // 编译阶段，可选，默认值见下 + "Compile" : {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, + // 运行阶段，必选 + "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, + // 答案检查阶段，可选，默认值见下 + "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} + } + }, + { + // C++ 语言 + "Lang" : "cpp", + // 使用默认评测脚本，答案比对方式为 NCMP(testlib)，Script 将被忽略 + "Judge" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + // 运行时配置：Run 必须存在，其余可选，默认值见 C 语言部分 + "Runtime": { + "Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1} + } + } + ], + // 评测点信息，总分应当为 100 分 + "Tasks" : [ + // 第一个评测点，分值 10 分，使用 ./data/{input,output}/1.{input,output} 为测试数据 + {"Id": 1, "Points": 10}, {"Id": 2, "Points": 20}, {"Id": 3, "Points": 30}, {"Id": 4, "Points": 40} diff --git a/resource/runner/problem/example/config.json b/resource/runner/problem/example/config.json index 3e4effc..acb3087 100644 --- a/resource/runner/problem/example/config.json +++ b/resource/runner/problem/example/config.json @@ -1,14 +1,24 @@ { - "Runtime": { - "TimeLimit": 1000, - "MemoryLimit": 16, - "NProcLimit": 1 - }, "Languages": [ - {"Lang": "c", "Type": "custom", "Script": "XYZ.Makefile", "Cmp": ""}, - {"Lang": "cpp", "Type": "default", "Script": "", "Cmp": "NCMP"} + { + "Lang" : "c", + "Judge" : {"Type": "custom", "Script": "XYZ.Makefile", "Cmp": ""}, + "Runtime": { + "Prebuild": {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, + "Compile" : {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, + "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, + "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} + } + }, + { + "Lang" : "cpp", + "Judge" : {"Type": "default", "Script": "", "Cmp": "NCMP"}, + "Runtime": { + "Run": {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1} + } + } ], - "Tasks": [ + "Tasks" : [ {"Id": 1, "Points": 10}, {"Id": 2, "Points": 20}, {"Id": 3, "Points": 30}, diff --git a/resource/runner/problem/example/judge/XYZ.Makefile b/resource/runner/problem/example/judge/custom.Makefile similarity index 95% rename from resource/runner/problem/example/judge/XYZ.Makefile rename to resource/runner/problem/example/judge/custom.Makefile index 665e359..11096ea 100644 --- a/resource/runner/problem/example/judge/XYZ.Makefile +++ b/resource/runner/problem/example/judge/custom.Makefile @@ -3,7 +3,6 @@ include ${TEMPLATE}/c.mk ${TEMPLATE}/Judger.mk # 评测分四个阶段 # 1. prebuild: 用于提前生成测试数据、评测器、spj等工具，runner 只执行一次 -# 详细信息见 XYZ.Makefile # 2. compile: 用于编译用户提交的程序 # 目录映射情况： # /woj/problem/judge 映射到题目目录的 ./judge <-- Readonly @@ -30,8 +29,6 @@ include ${TEMPLATE}/c.mk ${TEMPLATE}/Judger.mk # TEST_NUM=... <-- 当前测试点编号 # CMP=... <-- 在 config.json 中配置的比较器，如 NCMP # 其余通用环境变量，详见 ubuntu-full.Dockerfile -# 执行限制： -# 目前版本硬编码限制：时间 60s，内存 256mb compile: $(CC) $(CFLAGS) -o $(PREFIX)/user/$(USER_PROG).out $(PREFIX)/user/$(USER_PROG).$(LANG) $(PREFIX)/problem/judge/gadget.c diff --git a/resource/runner/scripts/ubuntu-full.Dockerfile b/resource/runner/scripts/ubuntu-full.Dockerfile index 980c278..4b27c16 100644 --- a/resource/runner/scripts/ubuntu-full.Dockerfile +++ b/resource/runner/scripts/ubuntu-full.Dockerfile @@ -1,9 +1,17 @@ FROM docker.io/library/ubuntu:22.04 WORKDIR /woj -# Install dependencies -RUN apt-get update && apt-get upgrade -y && apt-get install -y gcc g++ clang make cmake autoconf m4 libtool gperf git parallel python3 wget && apt-get clean && rm -rf /var/lib/apt/lists -RUN wget https://go.dev/dl/go1.21.5.linux-amd64.tar.gz && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz && rm go1.21.5.linux-amd64.tar.gz +# Install dependencies & languages +RUN apt-get update && apt-get upgrade -y && apt-get install -y software-properties-common \ + && add-apt-repository ppa:pypy/ppa && apt-get update \ + && apt-get install -y \ + git parallel wget curl \ + gcc g++ clang make cmake autoconf m4 libtool gperf \ + python3 pypy3 \ + && apt-get clean && rm -rf /var/lib/apt/lists +RUN wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.21.6.linux-amd64.tar.gz && rm go1.21.6.linux-amd64.tar.gz +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH=/usr/local/go/bin:/root/.cargo/bin:$PATH # Copy source code RUN mkdir -p /woj/framework && mkdir -p /woj/problem From 83d3913ba738edf570f7753aa393a201cc26ca4e Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sat, 27 Jan 2024 23:07:14 +0800 Subject: [PATCH 18/24] feat: support more detailed problem config (#7) --- internal/api/runner/build.go | 14 +- internal/api/runner/judge.go | 32 ++-- internal/e/code.go | 2 + internal/service/runner/common.go | 41 ++--- internal/service/runner/compile.go | 53 +++--- internal/service/runner/config.go | 238 +++++++++++++++++-------- internal/service/runner/new_problem.go | 53 +++--- internal/service/runner/run_judge.go | 105 +++++------ internal/service/runner/service.go | 10 +- internal/service/runner/status.go | 18 +- 10 files changed, 330 insertions(+), 236 deletions(-) diff --git a/internal/api/runner/build.go b/internal/api/runner/build.go index 5d4348b..7cc13d5 100644 --- a/internal/api/runner/build.go +++ b/internal/api/runner/build.go @@ -6,6 +6,7 @@ import ( "fmt" "git.0x7f.app/WOJ/woj-server/internal/e" "git.0x7f.app/WOJ/woj-server/internal/model" + "git.0x7f.app/WOJ/woj-server/internal/service/runner" "github.com/hibiken/asynq" "go.uber.org/zap" "time" @@ -18,6 +19,7 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { } h.log.Debug("build", zap.Any("payload", p)) + meta := runner.JudgeMeta{Version: p.ProblemVersionID} status, ctx := func() (e.Status, string) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) @@ -25,16 +27,20 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { return e.InternalError, "{\"Message\": \"storage error\"}" } - config, status := h.runnerService.NewProblem(p.ProblemVersionID, url, true) + config, status := h.runnerService.NewProblem(&meta, url, true) if status != e.Success { return e.InternalError, "{\"Message\": \"build error: " + status.String() + "\"}" } for i := range config.Languages { // do not store in db - config.Languages[i].Type = "" - config.Languages[i].Script = "" - config.Languages[i].Cmp = "" + config.Languages[i].Judge.Type = "" + config.Languages[i].Judge.Script = "" + config.Languages[i].Judge.Cmp = "" + + // Compile and Run is ok + config.Languages[i].Runtime.Prebuild = runner.ConfigRuntime{} + config.Languages[i].Runtime.Check = runner.ConfigRuntime{} } b, _ := json.Marshal(config) diff --git a/internal/api/runner/judge.go b/internal/api/runner/judge.go index 83a2268..5e015d0 100644 --- a/internal/api/runner/judge.go +++ b/internal/api/runner/judge.go @@ -23,50 +23,52 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { user := utils.RandomString(16) h.log.Debug("judge", zap.Any("payload", p), zap.String("user", user)) + meta := runner.JudgeMeta{Version: p.ProblemVersionID, User: user, Lang: p.Submission.Language} - status, point, ctx := func() (e.Status, int32, runner.JudgeStatus) { - systemError := runner.JudgeStatus{ - Message: "System Error", - Tasks: []runner.TaskStatus{{Verdict: runner.VerdictSystemError, Message: "API Error"}}, - } + SystemError := &runner.JudgeStatus{ + Message: "System Error", + Tasks: []runner.TaskStatus{{Verdict: runner.VerdictSystemError, Message: "API Error"}}, + } + + status, point, ctx := func() (e.Status, int32, *runner.JudgeStatus) { // 1. write user code userCode := filepath.Join(runner.UserDir, user, fmt.Sprintf("%s.%s", user, p.Submission.Language)) if !file.Touch(userCode) { - return e.InternalError, 0, systemError + return e.InternalError, 0, SystemError } err := file.Write(userCode, []byte(p.Submission.Code)) if err != nil { - return e.InternalError, 0, systemError + return e.InternalError, 0, SystemError } // 2. check problem - if !h.runnerService.ProblemExists(p.ProblemVersionID) { + if !h.runnerService.ProblemExists(&meta) { url, status := h.storageService.Get(p.StorageKey, time.Second*60*5) if status != e.Success { - return status, 0, systemError + return status, 0, SystemError } - _, status = h.runnerService.NewProblem(p.ProblemVersionID, url, false) + _, status = h.runnerService.NewProblem(&meta, url, false) if status != e.Success { - return status, 0, systemError + return status, 0, SystemError } } // 3. compile - compileResult, status := h.runnerService.Compile(p.ProblemVersionID, user, p.Submission.Language) + compileResult, status := h.runnerService.Compile(&meta) if status != e.Success { return status, 0, compileResult } // 4. run and judge - result, point, status := h.runnerService.RunAndJudge(p.ProblemVersionID, user, p.Submission.Language) + result, point, status := h.runnerService.RunAndJudge(&meta) return status, point, result }() if status == e.InternalError { // notice asynq to retry - return fmt.Errorf("internal error, ctx: %v", ctx) + return fmt.Errorf("internal error, ctx: %v", *ctx) } h.taskService.SubmitUpdate(&model.SubmitUpdatePayload{ @@ -75,7 +77,7 @@ func (h *handler) Judge(_ context.Context, t *asynq.Task) error { ProblemVersionID: p.ProblemVersionID, UserDir: user, Point: point, - }, ctx) + }, *ctx) return nil } diff --git a/internal/e/code.go b/internal/e/code.go index 9fa3e4e..78ef5c6 100644 --- a/internal/e/code.go +++ b/internal/e/code.go @@ -63,6 +63,7 @@ const ( RunnerUserCompileFailed RunnerRunFailed RunnerJudgeFailed + RunnerLanguageNotSupported ) const ( @@ -123,6 +124,7 @@ var msgText = map[Status]string{ RunnerUserCompileFailed: "Runner User Compile Failed", RunnerRunFailed: "Runner Run Failed", RunnerJudgeFailed: "Runner Judge Failed", + RunnerLanguageNotSupported: "Runner Language Not Supported", StorageUploadFailed: "Storage Upload Failed", StorageGetFailed: "Storage Get Failed", diff --git a/internal/service/runner/common.go b/internal/service/runner/common.go index 0f523ad..e76abc7 100644 --- a/internal/service/runner/common.go +++ b/internal/service/runner/common.go @@ -22,6 +22,12 @@ const ( ContainerImageRun = "git.0x7f.app/woj/ubuntu-run:latest" ) +type JudgeMeta struct { + Version uint + User string + Lang string +} + func init() { wd, err := os.Getwd() if err != nil { @@ -34,39 +40,34 @@ func init() { TmpDir = path.Join(Prefix, TmpDir) } -func (s *service) ProblemExists(version uint) bool { - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) +func (s *service) ProblemExists(meta *JudgeMeta) bool { + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) return file.Exist(problemPath) } -func (s *service) ValidatePath(version uint, user string, lang string) e.Status { - if !s.ProblemExists(version) { - s.log.Info("problem not exists", zap.Uint("version", version)) +func (s *service) ValidatePath(meta *JudgeMeta) e.Status { + if !s.ProblemExists(meta) { + s.log.Info("problem not exists", zap.Uint("version", meta.Version)) return e.RunnerProblemNotExist } - userPath := filepath.Join(UserDir, user, fmt.Sprintf("%s.%s", user, lang)) + userPath := filepath.Join(UserDir, meta.User, fmt.Sprintf("%s.%s", meta.User, meta.Lang)) if !file.Exist(userPath) { - s.log.Info("user program not exists", zap.String("user", user), zap.String("lang", lang)) + s.log.Info("user program not exists", zap.String("user", meta.User), zap.String("lang", meta.Lang)) return e.RunnerUserNotExist } return e.Success } -func (s *service) GetLangInfo(config *Config, lang string) (configLanguage, bool) { - for _, l := range config.Languages { - if l.Lang == lang { - return l, true - } +func (s *service) GetConfig(meta *JudgeMeta, skipCheck bool) (*Config, *ConfigLanguage, e.Status) { + config, err := s.ParseConfig(meta, skipCheck) + if err != nil { + return nil, nil, e.RunnerProblemParseFailed } - return configLanguage{}, false -} - -func (s *service) GetLangScript(l *configLanguage, lang string) string { - if l.Type == "default" { - return "/woj/framework/template/default/" + lang + ".Makefile" - } else { - return "/woj/problem/judge/" + l.Script + cLang, ok := config.FilterLanguage(meta.Lang) + if !ok { + return nil, nil, e.RunnerLanguageNotSupported } + return config, cLang, e.Success } diff --git a/internal/service/runner/compile.go b/internal/service/runner/compile.go index 919c4ff..5035dbe 100644 --- a/internal/service/runner/compile.go +++ b/internal/service/runner/compile.go @@ -13,34 +13,34 @@ import ( "time" ) -func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, e.Status) { +func (s *service) Compile(meta *JudgeMeta) (*JudgeStatus, e.Status) { // 1. ensure problem/user exists - status := s.ValidatePath(version, user, lang) + status := s.ValidatePath(meta) if status != e.Success { - return JudgeStatus{Message: "check failed"}, status + return &JudgeStatus{Message: "check failed"}, status } - config, err := s.ParseConfig(version, true) - if err != nil { - s.log.Error("[compile] parse config failed", zap.Error(err)) - return JudgeStatus{ + config, cLang, status := s.GetConfig(meta, true) + if status != e.Success { + s.log.Error("[compile] parse config failed", zap.Any("meta", *meta)) + return &JudgeStatus{ Message: "parse config failed", Tasks: []TaskStatus{{Verdict: VerdictSystemError, Message: "parse config failed"}}, }, e.RunnerProblemParseFailed } // 2. prepare judge environment - workDir := filepath.Join(UserDir, user) - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") + workDir := filepath.Join(UserDir, meta.User) + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") - sourceFile := filepath.Join(workDir, fmt.Sprintf("%s.%s", user, lang)) - targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", user)) + sourceFile := filepath.Join(workDir, fmt.Sprintf("%s.%s", meta.User, meta.Lang)) + targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", meta.User)) - logFile := filepath.Join(workDir, fmt.Sprintf("%s.compile.log", user)) + logFile := filepath.Join(workDir, fmt.Sprintf("%s.compile.log", meta.User)) log, err := os.Create(logFile) if err != nil { s.log.Error("[compile] create log file failed", zap.Error(err)) - return JudgeStatus{ + return &JudgeStatus{ Message: "create log file failed", Tasks: []TaskStatus{{Verdict: VerdictSystemError, Message: "create log file failed"}}, }, e.RunnerUserCompileFailed @@ -54,28 +54,28 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, DoAny(func() error { return os.Remove(targetFile) }). Do(func() error { return file.TouchErr(targetFile) }). Do(func() error { - l, ok := s.GetLangInfo(&config, lang) - script := s.GetLangScript(&l, lang) + l, ok := config.FilterLanguage(meta.Lang) if !ok { return e.RunnerProblemParseFailed.AsError() } + script := l.JudgeScript() args := &RunArgs{ Program: ProgramArgs{ Args: []string{"sh", "-c", fmt.Sprintf("cd /woj/user && make -f %s compile", script)}, - Env: []string{fmt.Sprintf("USER_PROG=%s", user), fmt.Sprintf("LANG=%s", lang)}, + Env: []string{fmt.Sprintf("USER_PROG=%s", meta.User), fmt.Sprintf("LANG=%s", meta.Lang)}, }, Runtime: RuntimeArgs{ Image: ContainerImageFull, - Memory: 256 * 1024 * 1024, // 256 MB - Timeout: time.Minute, + Pid: int64(cLang.Runtime.Compile.NProcLimit + 2), // bash + make + Memory: uint64(cLang.Runtime.Compile.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Compile.TimeLimit+1000)/1000) * time.Second, }, IO: IOArgs{ Output: log, Limit: 4 * 1024, // 4 KB }, } - args.Runtime.Mount = []specs.Mount{ { Source: judgeDir, @@ -85,13 +85,13 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, }, { Source: sourceFile, - Destination: fmt.Sprintf("/woj/user/%s.%s", user, lang), + Destination: fmt.Sprintf("/woj/user/%s.%s", meta.User, meta.Lang), Type: "bind", Options: []string{"rbind", "ro"}, }, { Source: targetFile, - Destination: fmt.Sprintf("/woj/user/%s.out", user), + Destination: fmt.Sprintf("/woj/user/%s.out", meta.User), Type: "bind", Options: []string{"rbind"}, }, @@ -102,12 +102,7 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, Done() if err != nil { - s.log.Info("[compile] compile failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[compile] compile failed", zap.Error(err), zap.Any("meta", *meta)) status = e.RunnerUserCompileFailed } @@ -118,11 +113,11 @@ func (s *service) Compile(version uint, user string, lang string) (JudgeStatus, msgText := string(msg) if !file.Exist(targetFile) || file.Empty(targetFile) { - return JudgeStatus{ + return &JudgeStatus{ Message: "compile failed", Tasks: []TaskStatus{{Verdict: VerdictCompileError, Message: msgText}}}, utils.If(status == e.Success, e.RunnerUserCompileFailed, status) } - return JudgeStatus{}, e.Success + return &JudgeStatus{}, e.Success } diff --git a/internal/service/runner/config.go b/internal/service/runner/config.go index d51ac3a..92cd0cd 100644 --- a/internal/service/runner/config.go +++ b/internal/service/runner/config.go @@ -4,106 +4,146 @@ import ( "encoding/json" "errors" "fmt" + "git.0x7f.app/WOJ/woj-server/pkg/utils" "os" "path/filepath" ) -type configLanguage struct { - Lang string `json:"Lang"` - Type string `json:"Type,omitempty"` - Script string `json:"Script,omitempty"` - Cmp string `json:"Cmp,omitempty"` +type ConfigRuntime struct { + TimeLimit int `json:"TimeLimit"` + MemoryLimit int `json:"MemoryLimit"` + NProcLimit int `json:"NProcLimit"` +} + +var ( + DefaultPrebuildRuntime = ConfigRuntime{ + TimeLimit: 300000, + MemoryLimit: 256, + NProcLimit: 64, + } + DefaultCompileRuntime = ConfigRuntime{ + TimeLimit: 60000, + MemoryLimit: 256, + NProcLimit: 64, + } + DefaultCheckRuntime = ConfigRuntime{ + TimeLimit: 60000, + MemoryLimit: 128, + NProcLimit: 64, + } +) + +func (c *ConfigRuntime) Validate() error { + if c.TimeLimit <= 0 { + return errors.New("time limit <= 0") + } + if c.MemoryLimit <= 0 { + return errors.New("memory limit <= 0") + } + if c.NProcLimit <= 0 { + return errors.New("nproc limit <= 0") + } + return nil +} + +type ConfigJudge struct { + Type string `json:"Type"` + Script string `json:"Script"` + Cmp string `json:"Cmp"` +} + +func (c *ConfigJudge) Validate(base string, lang string) error { + if c.Type != "custom" && c.Type != "default" { + return fmt.Errorf("language %s has invalid type %s", lang, c.Type) + } + + if c.Type == "custom" { + if c.Script == "" { + return fmt.Errorf("language %s has empty script", lang) + } + + file := filepath.Join(base, "judge", c.Script) + _, err := os.Stat(file) + if err != nil { + return fmt.Errorf("language %s has invalid script %s", lang, c.Script) + } + } + + if c.Type == "default" { + if c.Cmp == "" { + return fmt.Errorf("language %s has empty cmp", lang) + } + } + + return nil +} + +type ConfigLanguage struct { + Lang string `json:"Lang"` + Judge ConfigJudge `json:"Judge"` + Runtime struct { + Prebuild ConfigRuntime `json:"Prebuild"` + Compile ConfigRuntime `json:"Compile"` + Run ConfigRuntime `json:"Run"` + Check ConfigRuntime `json:"Check"` + } `json:"Runtime"` +} + +func (l *ConfigLanguage) JudgeScript() string { + if l.Judge.Type == "default" { + return "/woj/framework/template/default/" + l.Lang + ".Makefile" + } else { + return "/woj/problem/judge/" + l.Judge.Script + } } type Config struct { - Runtime struct { - TimeLimit int `json:"TimeLimit"` - MemoryLimit int `json:"MemoryLimit"` - NProcLimit int `json:"NProcLimit"` - } `json:"Runtime"` - Languages []configLanguage `json:"Languages"` + Languages []ConfigLanguage `json:"Languages"` Tasks []struct { Id int `json:"Id"` Points int32 `json:"Points"` } `json:"Tasks"` } -func (s *service) ParseConfig(version uint, skipCheck bool) (Config, error) { - base := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) - file := filepath.Join(base, "config.json") - - data, err := os.ReadFile(file) - if err != nil { - return Config{}, err - } - - config := Config{} - err = json.Unmarshal(data, &config) - if err != nil { - return Config{}, err - } - - if skipCheck { - return config, nil - } - - err = s.ValidateConfig(&config, base) - if err != nil { - return Config{}, err - } - - return config, nil -} - -func (s *service) ValidateConfig(config *Config, base string) error { - if config.Runtime.TimeLimit < 0 { - return errors.New("time limit is negative") - } - if config.Runtime.MemoryLimit < 0 { - return errors.New("memory limit is negative") - } - if config.Runtime.NProcLimit < 0 { - return errors.New("nproc limit is negative") - } - +func (c *Config) Validate(base string) error { allowedLang := map[string]struct{}{ - "c": {}, - "cpp": {}, + "c": {}, + "cpp": {}, + "go": {}, + "rust": {}, + "python3": {}, + "pypy3": {}, } - for _, lang := range config.Languages { + + for _, lang := range c.Languages { + // check language if _, ok := allowedLang[lang.Lang]; !ok { return fmt.Errorf("language %s is not allowed", lang.Lang) } - if lang.Type != "custom" && lang.Type != "default" { - return fmt.Errorf("language %s has invalid type %s", lang.Lang, lang.Type) + err := utils.NewMust(). + // check judge config + Do(func() error { return lang.Judge.Validate(base, lang.Lang) }). + // check runtime limits + Do(func() error { return lang.Runtime.Prebuild.Validate() }). + Do(func() error { return lang.Runtime.Compile.Validate() }). + Do(func() error { return lang.Runtime.Run.Validate() }). + Do(func() error { return lang.Runtime.Check.Validate() }). + Done() + + if err != nil { + return err } - if lang.Type == "custom" { - if lang.Script == "" { - return fmt.Errorf("language %s has empty script", lang.Lang) - } - - file := filepath.Join(base, "judge", lang.Script) - _, err := os.Stat(file) - if err != nil { - return fmt.Errorf("language %s has invalid script %s", lang.Lang, lang.Script) - } - } - - if lang.Type == "default" { - if lang.Cmp == "" { - return fmt.Errorf("language %s has empty cmp", lang.Lang) - } - } } - if len(config.Tasks) == 0 { + if len(c.Tasks) == 0 { return errors.New("no tasks") } + ids := map[int]struct{}{} - total := (1 + len(config.Tasks)) * len(config.Tasks) / 2 - for _, task := range config.Tasks { + total := (1 + len(c.Tasks)) * len(c.Tasks) / 2 + for _, task := range c.Tasks { if task.Id <= 0 { return fmt.Errorf("task %d has non-positive id", task.Id) } @@ -125,3 +165,53 @@ func (s *service) ValidateConfig(config *Config, base string) error { return nil } + +func (c *Config) FilterLanguage(lang string) (*ConfigLanguage, bool) { + for idx := range c.Languages { + if c.Languages[idx].Lang == lang { + return &c.Languages[idx], true + } + } + return &ConfigLanguage{}, false +} + +func (s *service) ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) { + base := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) + file := filepath.Join(base, "config.json") + + data, err := os.ReadFile(file) + if err != nil { + return &Config{}, err + } + + // unmarshal + config := &Config{} + err = json.Unmarshal(data, config) + if err != nil { + return &Config{}, err + } + + // fill default + for idx := range config.Languages { + if config.Languages[idx].Runtime.Prebuild.Validate() != nil { + config.Languages[idx].Runtime.Prebuild = DefaultPrebuildRuntime + } + if config.Languages[idx].Runtime.Compile.Validate() != nil { + config.Languages[idx].Runtime.Compile = DefaultCompileRuntime + } + if config.Languages[idx].Runtime.Check.Validate() != nil { + config.Languages[idx].Runtime.Check = DefaultCheckRuntime + } + } + + if skipCheck { + return config, nil + } + + err = config.Validate(base) + if err != nil { + return &Config{}, err + } + + return config, nil +} diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index 687002e..d75ec28 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -13,9 +13,9 @@ import ( "time" ) -func (s *service) DownloadProblem(version uint, url string) e.Status { - zipPath := filepath.Join(TmpDir, fmt.Sprintf("%d.zip", version)) - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) +func (s *service) DownloadProblem(meta *JudgeMeta, url string) e.Status { + zipPath := filepath.Join(TmpDir, fmt.Sprintf("%d.zip", meta.Version)) + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) err := down.Down(zipPath, url) if err != nil { @@ -32,26 +32,32 @@ func (s *service) DownloadProblem(version uint, url string) e.Status { return e.Success } -func (s *service) PrebuildProblem(version uint, force bool) e.Status { - if !s.ProblemExists(version) { +func (s *service) PrebuildProblem(meta *JudgeMeta, force bool) e.Status { + if !s.ProblemExists(meta) { return e.RunnerProblemNotExist } - mark := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), ".mark.prebuild") + mark := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), ".mark.prebuild") if force { _ = os.Remove(mark) } else if file.Exist(mark) { return e.Success } - prebuildScript := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge", "prebuild.Makefile") + prebuildScript := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge", "prebuild.Makefile") if !file.Exist(prebuildScript) { - s.log.Info("[new] prebuild script not found", zap.String("path", prebuildScript), zap.Uint("version", version)) + s.log.Info("[new] prebuild script not found", zap.String("path", prebuildScript), zap.Uint("version", meta.Version)) return e.Success } - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") + _, cLang, status := s.GetConfig(meta, false) + if status != e.Success { + s.log.Error("[new] parse config failed", zap.Any("meta", *meta)) + return e.RunnerProblemParseFailed + } + + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data") + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") args := &RunArgs{ Program: ProgramArgs{ @@ -59,8 +65,9 @@ func (s *service) PrebuildProblem(version uint, force bool) e.Status { }, Runtime: RuntimeArgs{ Image: ContainerImageFull, - Memory: 256 * 1024 * 1024, // 256MB - Timeout: 5 * time.Minute, + Pid: int64(cLang.Runtime.Prebuild.NProcLimit + 3), // sh + bash + make + Memory: uint64(cLang.Runtime.Prebuild.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Prebuild.TimeLimit+1000)/1000) * time.Second, }, } args.Runtime.Mount = []specs.Mount{ @@ -81,35 +88,35 @@ func (s *service) PrebuildProblem(version uint, force bool) e.Status { err := s.ContainerRun(args) if err != nil { - s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", version)) + s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", meta.Version)) return e.RunnerProblemPrebuildFailed } return e.Success } -func (s *service) NewProblem(version uint, url string, force bool) (Config, e.Status) { +func (s *service) NewProblem(meta *JudgeMeta, url string, force bool) (*Config, e.Status) { if force { - problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", version)) + problemPath := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version)) _ = os.RemoveAll(problemPath) } - if !s.ProblemExists(version) { - status := s.DownloadProblem(version, url) + if !s.ProblemExists(meta) { + status := s.DownloadProblem(meta, url) if status != e.Success { - return Config{}, status + return &Config{}, status } } - cfg, err := s.ParseConfig(version, false) + cfg, err := s.ParseConfig(meta, false) if err != nil { - s.log.Info("[new] parse problem failed", zap.Error(err), zap.Uint("version", version)) - return Config{}, e.RunnerProblemParseFailed + s.log.Info("[new] parse problem failed", zap.Error(err), zap.Uint("version", meta.Version)) + return &Config{}, e.RunnerProblemParseFailed } - status := s.PrebuildProblem(version, true) + status := s.PrebuildProblem(meta, true) if status != e.Success { - return Config{}, status + return &Config{}, status } return cfg, e.Success diff --git a/internal/service/runner/run_judge.go b/internal/service/runner/run_judge.go index c1cbf36..761f4f1 100644 --- a/internal/service/runner/run_judge.go +++ b/internal/service/runner/run_judge.go @@ -13,37 +13,43 @@ import ( "time" ) -func (s *service) SandboxArgsBuilder(config *Config, user string, lang string, id int) string { +func (s *service) SandboxArgsBuilder(meta *JudgeMeta, cLang *ConfigLanguage, id int) string { var args []string - args = append(args, fmt.Sprintf("--memory_limit=%d", config.Runtime.MemoryLimit)) - args = append(args, fmt.Sprintf("--nproc_limit=%d", config.Runtime.NProcLimit)) - args = append(args, fmt.Sprintf("--time_limit=%d", config.Runtime.TimeLimit)) - args = append(args, fmt.Sprintf("--sandbox_template=%s", lang)) + args = append(args, fmt.Sprintf("--memory_limit=%d", cLang.Runtime.Run.MemoryLimit)) + args = append(args, fmt.Sprintf("--nproc_limit=%d", cLang.Runtime.Run.NProcLimit)) + args = append(args, fmt.Sprintf("--time_limit=%d", cLang.Runtime.Run.TimeLimit)) + args = append(args, fmt.Sprintf("--sandbox_template=%s", cLang.Lang)) args = append(args, fmt.Sprintf("--sandbox_action=ret")) args = append(args, fmt.Sprintf("--uid=1000")) args = append(args, fmt.Sprintf("--gid=1000")) args = append(args, fmt.Sprintf("--file_input=/woj/problem/data/input/%d.input", id)) args = append(args, fmt.Sprintf("--file_output=/woj/user/%d.out.usr", id)) args = append(args, fmt.Sprintf("--file_info=/woj/user/%d.info", id)) - args = append(args, fmt.Sprintf("--program=/woj/user/%s.out", user)) + args = append(args, fmt.Sprintf("--program=/woj/user/%s.out", meta.User)) return strings.Join(args, " ") } -func (s *service) ProblemRun(version uint, user string, lang string, config *Config) { - workDir := filepath.Join(UserDir, user) - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data", "input") +func (s *service) ProblemRun(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) { + workDir := filepath.Join(UserDir, meta.User) + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data", "input") - // woj-sandbox killer will add 1 more second, here we add 1 also - timeout := time.Duration((config.Runtime.TimeLimit+1000)/1000+1+1) * time.Second + runtimeArgs := RuntimeArgs{ + Image: ContainerImageRun, + // sh, woj_launcher:program, woj_launcher:killer, woj_launcher:stat + Pid: int64(cLang.Runtime.Run.NProcLimit + 4), + Memory: uint64(cLang.Runtime.Run.MemoryLimit * 1024 * 1024), + // woj-sandbox killer will add 1 more second, here we add 1 also + Timeout: time.Duration((cLang.Runtime.Run.TimeLimit+1000)/1000+1+1) * time.Second, + } ids := make([]int, 0) for _, task := range config.Tasks { f := func(id int) func() { return func() { testCase := filepath.Join(dataDir, fmt.Sprintf("%d.input", id)) - targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", user)) + targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", meta.User)) ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) ifoFile := filepath.Join(workDir, fmt.Sprintf("%d.info", id)) @@ -52,16 +58,10 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con Args: []string{ "sh", "-c", "cd /woj/user && /woj/framework/scripts/woj_launcher " + - s.SandboxArgsBuilder(config, user, lang, id), + s.SandboxArgsBuilder(meta, cLang, id), }, }, - Runtime: RuntimeArgs{ - Image: ContainerImageRun, - // sh, woj_launcher:program, woj_launcher:killer, woj_launcher:stat - Pid: int64(config.Runtime.NProcLimit + 4), - Memory: uint64(config.Runtime.MemoryLimit * 1024 * 1024), - Timeout: timeout, - }, + Runtime: runtimeArgs, } args.Runtime.Mount = []specs.Mount{ { @@ -72,7 +72,7 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con }, { Source: targetFile, - Destination: fmt.Sprintf("/woj/user/%s.out", user), + Destination: fmt.Sprintf("/woj/user/%s.out", meta.User), Type: "bind", Options: []string{"rbind", "ro"}, }, @@ -99,12 +99,7 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con Done() if err != nil { - s.log.Info("[run] run failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[run] run failed", zap.Error(err), zap.Any("meta", *meta)) } } }(task.Id) @@ -118,10 +113,18 @@ func (s *service) ProblemRun(version uint, user string, lang string, config *Con } } -func (s *service) ProblemJudge(version uint, user string, lang string, config *Config) { - workDir := filepath.Join(UserDir, user) - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "data") - judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", version), "judge") +func (s *service) ProblemJudge(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) { + workDir := filepath.Join(UserDir, meta.User) + dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data") + judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") + script := cLang.JudgeScript() + + runtimeArgs := RuntimeArgs{ + Image: ContainerImageFull, + Pid: int64(cLang.Runtime.Check.NProcLimit + 2), // bash + make + Memory: uint64(cLang.Runtime.Check.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((cLang.Runtime.Check.TimeLimit+1000)/1000) * time.Second, + } ids := make([]int, 0) for _, task := range config.Tasks { @@ -130,25 +133,18 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) jdgFile := filepath.Join(workDir, fmt.Sprintf("%d.judge", id)) - c, ok := s.GetLangInfo(config, lang) - if !ok { - return - } - args := &RunArgs{ Program: ProgramArgs{ Args: []string{ "sh", "-c", - fmt.Sprintf("cd /woj/user && make -f %s judge", s.GetLangScript(&c, lang)), + fmt.Sprintf("cd /woj/user && make -f %s judge", script), }, Env: []string{ fmt.Sprintf("TEST_NUM=%d", id), - fmt.Sprintf("CMP=%s", c.Cmp), + fmt.Sprintf("CMP=%s", cLang.Judge.Cmp), }, }, - Runtime: RuntimeArgs{ - Image: ContainerImageFull, - }, + Runtime: runtimeArgs, } args.Runtime.Mount = []specs.Mount{ { @@ -184,12 +180,7 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C Done() if err != nil { - s.log.Info("[judge] judge failed", - zap.Error(err), - zap.Uint("version", version), - zap.String("user", user), - zap.String("lang", lang), - ) + s.log.Info("[judge] judge failed", zap.Error(err), zap.Any("meta", *meta)) } } }(task.Id) @@ -203,27 +194,27 @@ func (s *service) ProblemJudge(version uint, user string, lang string, config *C } } -func (s *service) RunAndJudge(version uint, user string, lang string) (JudgeStatus, int32, e.Status) { +func (s *service) RunAndJudge(meta *JudgeMeta) (*JudgeStatus, int32, e.Status) { // 1. ensure problem/user exists - status := s.ValidatePath(version, user, lang) + status := s.ValidatePath(meta) if status != e.Success { - return JudgeStatus{Message: "check failed"}, 0, status + return &JudgeStatus{Message: "check failed"}, 0, status } // 2. config - config, err := s.ParseConfig(version, false) - if err != nil { - return JudgeStatus{Message: "parse config failed"}, 0, e.RunnerProblemParseFailed + config, cLang, status := s.GetConfig(meta, true) + if status != e.Success { + return &JudgeStatus{Message: status.String()}, 0, status } // 3. run user program - s.ProblemRun(version, user, lang, &config) + s.ProblemRun(meta, config, cLang) - // 4. run judger - s.ProblemJudge(version, user, lang, &config) + // 4. run judge + s.ProblemJudge(meta, config, cLang) // 5. check result - result, pts := s.checkResults(user, &config) + result, pts := s.CheckResults(meta, config, cLang) return result, pts, e.Success } diff --git a/internal/service/runner/service.go b/internal/service/runner/service.go index ab5c826..fca0f9d 100644 --- a/internal/service/runner/service.go +++ b/internal/service/runner/service.go @@ -21,17 +21,17 @@ type Service interface { // EnsureDeps build docker images EnsureDeps(force bool) e.Status // NewProblem = Download + Parse + Prebuild - NewProblem(version uint, url string, force bool) (Config, e.Status) + NewProblem(meta *JudgeMeta, url string, force bool) (*Config, e.Status) // Compile compile user submission - Compile(version uint, user string, lang string) (JudgeStatus, e.Status) + Compile(meta *JudgeMeta) (*JudgeStatus, e.Status) // RunAndJudge execute user program - RunAndJudge(version uint, user string, lang string) (JudgeStatus, int32, e.Status) + RunAndJudge(meta *JudgeMeta) (*JudgeStatus, int32, e.Status) // ParseConfig parse config file - ParseConfig(version uint, skipCheck bool) (Config, error) + ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) // ProblemExists check if problem exists - ProblemExists(version uint) bool + ProblemExists(meta *JudgeMeta) bool HealthCheck() error Shutdown() error diff --git a/internal/service/runner/status.go b/internal/service/runner/status.go index 95ddfd8..c800760 100644 --- a/internal/service/runner/status.go +++ b/internal/service/runner/status.go @@ -97,12 +97,12 @@ func (t *TaskStatus) checkExit() *TaskStatus { return t } -func (t *TaskStatus) checkTime(config *Config) *TaskStatus { +func (t *TaskStatus) checkTime(cLang *ConfigLanguage) *TaskStatus { if t.Verdict != VerdictAccepted { return t } - if t.info["real_time"].(float64) > float64(config.Runtime.TimeLimit)+5 { + if t.info["real_time"].(float64) > float64(cLang.Runtime.Run.TimeLimit)+5 { t.Verdict = VerdictTimeLimitExceeded t.Message = fmt.Sprintf("real_time: %v cpu_time: %v", t.info["real_time"], t.info["cpu_time"]) } @@ -110,12 +110,12 @@ func (t *TaskStatus) checkTime(config *Config) *TaskStatus { return t } -func (t *TaskStatus) checkMemory(config *Config) *TaskStatus { +func (t *TaskStatus) checkMemory(cLang *ConfigLanguage) *TaskStatus { if t.Verdict != VerdictAccepted { return t } - if t.info["memory"].(float64) > float64((config.Runtime.MemoryLimit+1)*1024) { + if t.info["memory"].(float64) > float64((cLang.Runtime.Run.MemoryLimit+1)*1024) { t.Verdict = VerdictMemoryLimitExceeded t.Message = fmt.Sprintf("memory: %v", t.info["memory"]) } @@ -193,7 +193,7 @@ func (t *TaskStatus) checkJudge(pts *map[int]int32) *TaskStatus { return t } -func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) { +func (s *service) CheckResults(meta *JudgeMeta, config *Config, cLang *ConfigLanguage) (*JudgeStatus, int32) { // CE will be processed in phase compile pts := map[int]int32{} @@ -202,7 +202,7 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) } var results []TaskStatus - dir := filepath.Join(UserDir, user) + dir := filepath.Join(UserDir, meta.User) var sum int32 = 0 for i := 1; i <= len(config.Tasks); i++ { @@ -213,8 +213,8 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) result.getInfoText(info). getInfo(). - checkTime(config). - checkMemory(config). + checkTime(cLang). + checkMemory(cLang). checkExit(). getJudgeText(judge). getJudge(). @@ -224,5 +224,5 @@ func (s *service) checkResults(user string, config *Config) (JudgeStatus, int32) results = append(results, result) } - return JudgeStatus{Message: "", Tasks: results}, sum + return &JudgeStatus{Message: "", Tasks: results}, sum } From 8429988cb46817ec0aefa285e9ba27b9c76da5e6 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 18:15:07 +0800 Subject: [PATCH 19/24] fix: Prebuild should be independent of Languages --- internal/api/runner/build.go | 8 ++---- internal/service/runner/config.go | 27 ++++++++++++--------- internal/service/runner/new_problem.go | 16 ++++-------- resource/runner/problem/example/README.md | 10 ++++---- resource/runner/problem/example/config.json | 10 ++++---- 5 files changed, 33 insertions(+), 38 deletions(-) diff --git a/internal/api/runner/build.go b/internal/api/runner/build.go index 7cc13d5..c6749c7 100644 --- a/internal/api/runner/build.go +++ b/internal/api/runner/build.go @@ -34,12 +34,8 @@ func (h *handler) Build(_ context.Context, t *asynq.Task) error { for i := range config.Languages { // do not store in db - config.Languages[i].Judge.Type = "" - config.Languages[i].Judge.Script = "" - config.Languages[i].Judge.Cmp = "" - - // Compile and Run is ok - config.Languages[i].Runtime.Prebuild = runner.ConfigRuntime{} + config.Languages[i].Judge = runner.ConfigJudge{} + config.Prebuild = runner.ConfigRuntime{} config.Languages[i].Runtime.Check = runner.ConfigRuntime{} } diff --git a/internal/service/runner/config.go b/internal/service/runner/config.go index 92cd0cd..7e5ef5c 100644 --- a/internal/service/runner/config.go +++ b/internal/service/runner/config.go @@ -80,12 +80,11 @@ func (c *ConfigJudge) Validate(base string, lang string) error { type ConfigLanguage struct { Lang string `json:"Lang"` - Judge ConfigJudge `json:"Judge"` + Judge ConfigJudge `json:"Judge,omitempty"` Runtime struct { - Prebuild ConfigRuntime `json:"Prebuild"` - Compile ConfigRuntime `json:"Compile"` - Run ConfigRuntime `json:"Run"` - Check ConfigRuntime `json:"Check"` + Compile ConfigRuntime `json:"Compile"` + Run ConfigRuntime `json:"Run"` + Check ConfigRuntime `json:"Check,omitempty"` } `json:"Runtime"` } @@ -99,6 +98,7 @@ func (l *ConfigLanguage) JudgeScript() string { type Config struct { Languages []ConfigLanguage `json:"Languages"` + Prebuild ConfigRuntime `json:"Prebuild,omitempty"` Tasks []struct { Id int `json:"Id"` Points int32 `json:"Points"` @@ -115,8 +115,8 @@ func (c *Config) Validate(base string) error { "pypy3": {}, } + // check per language config for _, lang := range c.Languages { - // check language if _, ok := allowedLang[lang.Lang]; !ok { return fmt.Errorf("language %s is not allowed", lang.Lang) } @@ -125,7 +125,6 @@ func (c *Config) Validate(base string) error { // check judge config Do(func() error { return lang.Judge.Validate(base, lang.Lang) }). // check runtime limits - Do(func() error { return lang.Runtime.Prebuild.Validate() }). Do(func() error { return lang.Runtime.Compile.Validate() }). Do(func() error { return lang.Runtime.Run.Validate() }). Do(func() error { return lang.Runtime.Check.Validate() }). @@ -134,9 +133,15 @@ func (c *Config) Validate(base string) error { if err != nil { return err } - } + // check prebuild config + if err := c.Prebuild.Validate(); err != nil { + return err + } + + // check tasks + if len(c.Tasks) == 0 { return errors.New("no tasks") } @@ -193,9 +198,6 @@ func (s *service) ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) // fill default for idx := range config.Languages { - if config.Languages[idx].Runtime.Prebuild.Validate() != nil { - config.Languages[idx].Runtime.Prebuild = DefaultPrebuildRuntime - } if config.Languages[idx].Runtime.Compile.Validate() != nil { config.Languages[idx].Runtime.Compile = DefaultCompileRuntime } @@ -203,6 +205,9 @@ func (s *service) ParseConfig(meta *JudgeMeta, skipCheck bool) (*Config, error) config.Languages[idx].Runtime.Check = DefaultCheckRuntime } } + if config.Prebuild.Validate() != nil { + config.Prebuild = DefaultPrebuildRuntime + } if skipCheck { return config, nil diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index d75ec28..94fbb93 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -32,7 +32,7 @@ func (s *service) DownloadProblem(meta *JudgeMeta, url string) e.Status { return e.Success } -func (s *service) PrebuildProblem(meta *JudgeMeta, force bool) e.Status { +func (s *service) PrebuildProblem(meta *JudgeMeta, config *Config, force bool) e.Status { if !s.ProblemExists(meta) { return e.RunnerProblemNotExist } @@ -50,12 +50,6 @@ func (s *service) PrebuildProblem(meta *JudgeMeta, force bool) e.Status { return e.Success } - _, cLang, status := s.GetConfig(meta, false) - if status != e.Success { - s.log.Error("[new] parse config failed", zap.Any("meta", *meta)) - return e.RunnerProblemParseFailed - } - dataDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "data") judgeDir := filepath.Join(ProblemDir, fmt.Sprintf("%d", meta.Version), "judge") @@ -65,9 +59,9 @@ func (s *service) PrebuildProblem(meta *JudgeMeta, force bool) e.Status { }, Runtime: RuntimeArgs{ Image: ContainerImageFull, - Pid: int64(cLang.Runtime.Prebuild.NProcLimit + 3), // sh + bash + make - Memory: uint64(cLang.Runtime.Prebuild.MemoryLimit * 1024 * 1024), - Timeout: time.Duration((cLang.Runtime.Prebuild.TimeLimit+1000)/1000) * time.Second, + Pid: int64(config.Prebuild.NProcLimit + 3), // sh + bash + make + Memory: uint64(config.Prebuild.MemoryLimit * 1024 * 1024), + Timeout: time.Duration((config.Prebuild.TimeLimit+1000)/1000) * time.Second, }, } args.Runtime.Mount = []specs.Mount{ @@ -114,7 +108,7 @@ func (s *service) NewProblem(meta *JudgeMeta, url string, force bool) (*Config, return &Config{}, e.RunnerProblemParseFailed } - status := s.PrebuildProblem(meta, true) + status := s.PrebuildProblem(meta, cfg, true) if status != e.Success { return &Config{}, status } diff --git a/resource/runner/problem/example/README.md b/resource/runner/problem/example/README.md index 14f9f0d..f577fd5 100644 --- a/resource/runner/problem/example/README.md +++ b/resource/runner/problem/example/README.md @@ -34,14 +34,12 @@ "Judge" : {"Type": "custom", "Script": "custom.Makefile", "Cmp": ""}, // 运行时配置：时间(ms) 内存(MB) 进/线程数目 "Runtime": { - // 题目构建阶段，用于生成测试数据等，可选，默认值见下 - "Prebuild": {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, // 编译阶段，可选，默认值见下 - "Compile" : {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, + "Compile": {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, // 运行阶段，必选 - "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, + "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, // 答案检查阶段，可选，默认值见下 - "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} + "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} } }, { @@ -55,6 +53,8 @@ } } ], + // 题目构建阶段，用于生成测试数据等，可选，默认值见下 + "Prebuild" : {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, // 评测点信息，总分应当为 100 分 "Tasks" : [ // 第一个评测点，分值 10 分，使用 ./data/{input,output}/1.{input,output} 为测试数据 diff --git a/resource/runner/problem/example/config.json b/resource/runner/problem/example/config.json index acb3087..46bb174 100644 --- a/resource/runner/problem/example/config.json +++ b/resource/runner/problem/example/config.json @@ -2,12 +2,11 @@ "Languages": [ { "Lang" : "c", - "Judge" : {"Type": "custom", "Script": "XYZ.Makefile", "Cmp": ""}, + "Judge" : {"Type": "custom", "Script": "custom.Makefile", "Cmp": ""}, "Runtime": { - "Prebuild": {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, - "Compile" : {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, - "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, - "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} + "Compile": {"TimeLimit": 60000, "MemoryLimit": 256, "NProcLimit": 64}, + "Run" : {"TimeLimit": 1000, "MemoryLimit": 16, "NProcLimit": 1}, + "Check" : {"TimeLimit": 60000, "MemoryLimit": 128, "NProcLimit": 64} } }, { @@ -18,6 +17,7 @@ } } ], + "Prebuild" : {"TimeLimit": 300000, "MemoryLimit": 256, "NProcLimit": 64}, "Tasks" : [ {"Id": 1, "Points": 10}, {"Id": 2, "Points": 20}, From 6965435cb786e4a65c6c0c93a01d6806992860b4 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 18:16:04 +0800 Subject: [PATCH 20/24] feat: `compile` and `new_problem` are queued by pool (#7) --- internal/service/runner/compile.go | 3 +- internal/service/runner/container.go | 4 +++ internal/service/runner/new_problem.go | 3 +- internal/service/runner/run_judge.go | 14 ++++---- pkg/pool/pool.go | 45 +++++++++++++++----------- pkg/pool/pool_test.go | 26 ++++++++++----- pkg/pool/task.go | 2 +- pkg/pool/worker.go | 4 +-- 8 files changed, 63 insertions(+), 38 deletions(-) diff --git a/internal/service/runner/compile.go b/internal/service/runner/compile.go index 5035dbe..cf9662d 100644 --- a/internal/service/runner/compile.go +++ b/internal/service/runner/compile.go @@ -97,7 +97,8 @@ func (s *service) Compile(meta *JudgeMeta) (*JudgeStatus, e.Status) { }, } - return s.ContainerRun(args) + id := s.ContainerRunPool(args) + return s.pool.WaitForTask(id) }). Done() diff --git a/internal/service/runner/container.go b/internal/service/runner/container.go index 3be2367..917350d 100644 --- a/internal/service/runner/container.go +++ b/internal/service/runner/container.go @@ -136,3 +136,7 @@ func (s *service) ContainerRun(arg *RunArgs) error { return nil } + +func (s *service) ContainerRunPool(arg *RunArgs) int { + return s.pool.AddTask(func() error { return s.ContainerRun(arg) }) +} diff --git a/internal/service/runner/new_problem.go b/internal/service/runner/new_problem.go index 94fbb93..47e9d1b 100644 --- a/internal/service/runner/new_problem.go +++ b/internal/service/runner/new_problem.go @@ -79,7 +79,8 @@ func (s *service) PrebuildProblem(meta *JudgeMeta, config *Config, force bool) e }, } - err := s.ContainerRun(args) + id := s.ContainerRunPool(args) + err := s.pool.WaitForTask(id) if err != nil { s.log.Warn("[new] prebuild problem failed", zap.Error(err), zap.Uint("version", meta.Version)) diff --git a/internal/service/runner/run_judge.go b/internal/service/runner/run_judge.go index 761f4f1..ea826aa 100644 --- a/internal/service/runner/run_judge.go +++ b/internal/service/runner/run_judge.go @@ -46,8 +46,8 @@ func (s *service) ProblemRun(meta *JudgeMeta, config *Config, cLang *ConfigLangu ids := make([]int, 0) for _, task := range config.Tasks { - f := func(id int) func() { - return func() { + f := func(id int) func() error { + return func() error { testCase := filepath.Join(dataDir, fmt.Sprintf("%d.input", id)) targetFile := filepath.Join(workDir, fmt.Sprintf("%s.out", meta.User)) ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) @@ -101,6 +101,7 @@ func (s *service) ProblemRun(meta *JudgeMeta, config *Config, cLang *ConfigLangu if err != nil { s.log.Info("[run] run failed", zap.Error(err), zap.Any("meta", *meta)) } + return err } }(task.Id) @@ -109,7 +110,7 @@ func (s *service) ProblemRun(meta *JudgeMeta, config *Config, cLang *ConfigLangu } for _, id := range ids { - s.pool.WaitForTask(id) + _ = s.pool.WaitForTask(id) } } @@ -128,8 +129,8 @@ func (s *service) ProblemJudge(meta *JudgeMeta, config *Config, cLang *ConfigLan ids := make([]int, 0) for _, task := range config.Tasks { - f := func(id int) func() { - return func() { + f := func(id int) func() error { + return func() error { ansFile := filepath.Join(workDir, fmt.Sprintf("%d.out.usr", id)) jdgFile := filepath.Join(workDir, fmt.Sprintf("%d.judge", id)) @@ -182,6 +183,7 @@ func (s *service) ProblemJudge(meta *JudgeMeta, config *Config, cLang *ConfigLan if err != nil { s.log.Info("[judge] judge failed", zap.Error(err), zap.Any("meta", *meta)) } + return err } }(task.Id) @@ -190,7 +192,7 @@ func (s *service) ProblemJudge(meta *JudgeMeta, config *Config, cLang *ConfigLan } for _, id := range ids { - s.pool.WaitForTask(id) + _ = s.pool.WaitForTask(id) } } diff --git a/pkg/pool/pool.go b/pkg/pool/pool.go index f1a04ca..4106fdb 100644 --- a/pkg/pool/pool.go +++ b/pkg/pool/pool.go @@ -1,6 +1,7 @@ package pool import ( + "errors" "sync" ) @@ -11,14 +12,14 @@ type TaskPool struct { lck sync.Mutex curTaskID int - waitMap map[int]chan struct{} + waitMap map[int]chan error } func NewTaskPool(maxWorkers, bufferSize int) *TaskPool { return &TaskPool{ workers: maxWorkers, queue: make(chan Task, bufferSize), - waitMap: make(map[int]chan struct{}), + waitMap: make(map[int]chan error), curTaskID: 1, // task id starts from 1 } } @@ -31,13 +32,13 @@ func (tp *TaskPool) Start() { } } -func (tp *TaskPool) AddTask(f func()) int { +func (tp *TaskPool) AddTask(f func() error) int { tp.lck.Lock() id := tp.curTaskID tp.curTaskID++ - waitChan := make(chan struct{}) + waitChan := make(chan error, 1) tp.waitMap[id] = waitChan tp.lck.Unlock() @@ -48,7 +49,26 @@ func (tp *TaskPool) AddTask(f func()) int { return id } -func (tp *TaskPool) WaitForTask(taskID int) { +func (tp *TaskPool) WaitForTask(taskID int) error { + tp.lck.Lock() + waitChan, ok := tp.waitMap[taskID] + if !ok { + tp.lck.Unlock() + return errors.New("task not found") + } + tp.lck.Unlock() + + ret := <-waitChan + close(waitChan) + + tp.lck.Lock() + delete(tp.waitMap, taskID) + tp.lck.Unlock() + + return ret +} + +func (tp *TaskPool) markTaskComplete(taskID int, err error) { tp.lck.Lock() waitChan, ok := tp.waitMap[taskID] if !ok { @@ -57,20 +77,7 @@ func (tp *TaskPool) WaitForTask(taskID int) { } tp.lck.Unlock() - <-waitChan -} - -func (tp *TaskPool) markTaskComplete(taskID int) { - tp.lck.Lock() - defer tp.lck.Unlock() - - waitChan, ok := tp.waitMap[taskID] - if !ok { - return - } - - close(waitChan) - delete(tp.waitMap, taskID) + waitChan <- err } func (tp *TaskPool) Stop() { diff --git a/pkg/pool/pool_test.go b/pkg/pool/pool_test.go index f82680c..a06389e 100644 --- a/pkg/pool/pool_test.go +++ b/pkg/pool/pool_test.go @@ -1,6 +1,8 @@ package pool import ( + "errors" + "strconv" "sync" "testing" "time" @@ -14,8 +16,8 @@ func TestTaskPool_Stop(t *testing.T) { counter := 0 for i := 1; i <= 10; i++ { - f := func(i int) func() { - return func() { + f := func(i int) func() error { + return func() error { lck.Lock() t.Log("task", i, "locked") counter += i @@ -24,6 +26,8 @@ func TestTaskPool_Stop(t *testing.T) { time.Sleep(time.Duration(i*100) * time.Millisecond) t.Log("task", i, "finished") + + return nil } }(i) pool.AddTask(f) @@ -43,18 +47,22 @@ func TestTaskPool_WaitForTask(t *testing.T) { counter := 0 for i := 1; i <= 10; i++ { - f := func(i int) func() { - return func() { + f := func(i int) func() error { + return func() error { counter += 1 t.Log("task", i, "finished") + return errors.New(strconv.Itoa(i)) } }(i) id := pool.AddTask(f) - pool.WaitForTask(id) + ret := pool.WaitForTask(id) if counter != 1 { t.Errorf("Counter mismatch: expected %d, got %d, task %d", 1, counter, id) } + if ret.Error() != strconv.Itoa(i) { + t.Errorf("Return value mismatch: expected %s, got %s, task %d", strconv.Itoa(i), ret.Error(), id) + } counter -= 1 } @@ -70,8 +78,8 @@ func TestTaskPool_One(t *testing.T) { ids := make([]int, 0) for i := 1; i <= 10; i++ { - f := func(i int) func() { - return func() { + f := func(i int) func() error { + return func() error { lck.Lock() t.Log("task", i, "locked") counter += i @@ -80,6 +88,8 @@ func TestTaskPool_One(t *testing.T) { time.Sleep(time.Duration(i*10) * time.Millisecond) t.Log("task", i, "finished") + + return nil } }(i) id := pool.AddTask(f) @@ -87,7 +97,7 @@ func TestTaskPool_One(t *testing.T) { } for _, id := range ids { - pool.WaitForTask(id) + _ = pool.WaitForTask(id) } if counter != 55 { diff --git a/pkg/pool/task.go b/pkg/pool/task.go index d3f8b7c..7efa300 100644 --- a/pkg/pool/task.go +++ b/pkg/pool/task.go @@ -2,5 +2,5 @@ package pool type Task struct { id int - f func() + f func() error } diff --git a/pkg/pool/worker.go b/pkg/pool/worker.go index 8cd5473..55e8fef 100644 --- a/pkg/pool/worker.go +++ b/pkg/pool/worker.go @@ -18,7 +18,7 @@ func (w *Worker) Start(wg *sync.WaitGroup) { defer wg.Done() for task := range w.queue { - task.f() - w.pool.markTaskComplete(task.id) + err := task.f() + w.pool.markTaskComplete(task.id, err) } } From 23f6012f8cdd4e7641bf9e0f674a646ade41d5a9 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 21:08:13 +0800 Subject: [PATCH 21/24] fix: add missing volumes in docker-compose.yml --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 519e792..daa9db3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,6 +94,7 @@ services: volumes: runner: + container: storage: cache: db: From 6e22df8dda50b93c5c438b96b23c0f9b3b7d6fe7 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 21:25:08 +0800 Subject: [PATCH 22/24] chore: update Dockerfile: sync with frontend changes --- Runner.Dockerfile | 2 +- Server.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Runner.Dockerfile b/Runner.Dockerfile index 2072f2a..389ec00 100644 --- a/Runner.Dockerfile +++ b/Runner.Dockerfile @@ -1,7 +1,7 @@ # builder FROM docker.io/library/golang:alpine AS builder -ENV GOPROXY=https://goproxy.cn +#ENV GOPROXY=https://goproxy.cn ENV CGO_ENABLED=0 WORKDIR /builder diff --git a/Server.Dockerfile b/Server.Dockerfile index ac36edc..260feec 100644 --- a/Server.Dockerfile +++ b/Server.Dockerfile @@ -16,7 +16,7 @@ COPY . /builder RUN --mount=type=cache,id=golang,target=/go/pkg make build # UI Builder -FROM git.0x7f.app/woj/woj-ui:1.0.0 AS ui-builder +FROM git.0x7f.app/woj/woj-ui:1.1.0 AS ui-builder RUN find /app -type f -name "*.map" -delete # main image From 170cdd189896f1a6621c520f0e3bdda3d379ee8d Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 21:34:43 +0800 Subject: [PATCH 23/24] chore: update deps --- go.sum | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go.sum b/go.sum index 8785fee..95dcc1b 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,7 @@ github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLI github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= @@ -51,6 +52,7 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo= @@ -227,6 +229,7 @@ github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiw github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= @@ -307,6 +310,7 @@ github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4 h1:E github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= @@ -343,6 +347,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU= @@ -386,6 +392,7 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= From caaccd6e30f7af92092ea917bcbe2a982e09ca13 Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Sun, 28 Jan 2024 21:35:03 +0800 Subject: [PATCH 24/24] release: 1.3.0 --- VERSION | 2 +- resource/deploy/runner.yaml | 9 ++++++++- resource/deploy/server.yaml | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 0d9c5e7..589268e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.3-dev \ No newline at end of file +1.3.0 \ No newline at end of file diff --git a/resource/deploy/runner.yaml b/resource/deploy/runner.yaml index 8f2bc47..aaf6916 100644 --- a/resource/deploy/runner.yaml +++ b/resource/deploy/runner.yaml @@ -35,7 +35,7 @@ spec: spec: containers: - name: runner - image: git.0x7f.app/woj/woj-runner:1.2.2 + image: git.0x7f.app/woj/woj-runner:1.3.0 imagePullPolicy: IfNotPresent args: - runner @@ -61,12 +61,19 @@ spec: key: MINIO_ROOT_PASSWORD - name: STORAGE_BUCKET value: "woj" + - name: START_CONTAINERD + value: "true" securityContext: privileged: true volumeMounts: - name: runner-vol mountPath: /app/resource/runner/user + - name: container-vol + mountPath: /var/lib/containerd volumes: - name: runner-vol persistentVolumeClaim: claimName: runner-pvc + - name: container-vol + persistentVolumeClaim: + claimName: runner-pvc diff --git a/resource/deploy/server.yaml b/resource/deploy/server.yaml index 5432a6d..d37cfdd 100644 --- a/resource/deploy/server.yaml +++ b/resource/deploy/server.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: server - image: git.0x7f.app/woj/woj-server:1.2.2 + image: git.0x7f.app/woj/woj-server:1.3.0 imagePullPolicy: IfNotPresent args: - server