feat(backend): propagate otel trace through queued jobs

## Problem
MMseqs2-App lost request trace context as soon as work was enqueued, so
request-side and worker-side logs could not be correlated reliably. The
existing plain-text logs also only exposed `traceparent`, which is harder to
parse in systems like SigNoz or Datadog.

## Solution
Add an OTEL trace context payload to `JobRequest`, populate it once from HTTP
middleware, and pass it through the job constructors so the same context is
available when the worker executes the job. Update job-scoped worker logs to
append that context, and include explicit `trace_id` and `span_id` fields
derived from `traceparent` for easier downstream parsing.

Author: simonepri

backend/complexsearchjob.go

@@ -48,7 +48,7 @@ func (r ComplexSearchJob) WritePDB(path string) error {
 	return nil
 }
 
-func NewComplexSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string) (JobRequest, error) {
+func NewComplexSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := ComplexSearchJob{
 		max(strings.Count(query, "HEADER"), 1),
 		dbs,
@@ -58,11 +58,12 @@ func NewComplexSearchJobRequest(query string, dbs []string, validDbs []Params, m
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobComplexSearch,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobComplexSearch,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	ids := make([]string, 0)

backend/folddiscojob.go

@@ -53,7 +53,7 @@ func (r FoldDiscoJob) WritePDB(path string) error {
 	return nil
 }
 
-func NewFoldDiscoJobRequest(query string, motif string, dbs []string, validDbs []Params, resultPath string, email string) (JobRequest, error) {
+func NewFoldDiscoJobRequest(query string, motif string, dbs []string, validDbs []Params, resultPath string, email string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := FoldDiscoJob{
 		max(strings.Count(query, "HEADER"), 1),
 		dbs,
@@ -64,11 +64,12 @@ func NewFoldDiscoJobRequest(query string, motif string, dbs []string, validDbs [
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobFoldDisco,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobFoldDisco,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	ids := make([]string, 0)

backend/foldmasonmsa.go

@@ -77,17 +77,19 @@ func NewFoldMasonMSAJobRequest(
 	fileNames []string,
 	gapOpen int64,
 	gapExtend int64,
+	otelTrace OtelTraceContext,
 ) (JobRequest, error) {
 	job := FoldMasonMSAJob{
 		queries,
 		fileNames,
 	}
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobFoldMasonMSA,
-		job,
-		"",
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobFoldMasonMSA,
+		Job:       job,
+		Email:     "",
+		OtelTrace: otelTrace.Ptr(),
 	}
 	return request, nil
 }

backend/indexjob.go

@@ -22,17 +22,18 @@ func (r IndexJob) Rank() float64 {
 	return float64(0)
 }
 
-func NewIndexJobRequest(path string, email string) (JobRequest, error) {
+func NewIndexJobRequest(path string, email string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := IndexJob{
 		path,
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobIndex,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobIndex,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	return request, nil

backend/interfacesearchjob.go

@@ -48,7 +48,7 @@ func (r InterfaceSearchJob) WritePDB(path string) error {
 	return nil
 }
 
-func NewInterfaceSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string) (JobRequest, error) {
+func NewInterfaceSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := InterfaceSearchJob{
 		max(strings.Count(query, "HEADER"), 1),
 		dbs,
@@ -58,11 +58,12 @@ func NewInterfaceSearchJobRequest(query string, dbs []string, validDbs []Params,
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobInterfaceSearch,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobInterfaceSearch,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	ids := make([]string, 0)

backend/jobsystem.go

@@ -31,11 +31,19 @@ const (
 )
 
 type JobRequest struct {
-	Id     Id          `json:"id" validate:"required"`
-	Status Status      `json:"status" validate:"required"`
-	Type   JobType     `json:"type" validate:"required"`
-	Job    interface{} `json:"job" validate:"required"`
-	Email  string      `json:"email" validate:"omitempty,email"`
+	Id        Id                `json:"id" validate:"required"`
+	Status    Status            `json:"status" validate:"required"`
+	Type      JobType           `json:"type" validate:"required"`
+	Job       interface{}       `json:"job" validate:"required"`
+	Email     string            `json:"email" validate:"omitempty,email"`
+	OtelTrace *OtelTraceContext `json:"otelTrace,omitempty"`
+}
+
+func (r JobRequest) String() string {
+	if r.OtelTrace == nil {
+		return fmt.Sprintf("ticket=%s type=%s status=%s", r.Id, r.Type, r.Status)
+	}
+	return fmt.Sprintf("ticket=%s type=%s status=%s%s", r.Id, r.Type, r.Status, *r.OtelTrace)
 }
 
 type jobRequest JobRequest

backend/msajob.go

@@ -43,7 +43,7 @@ func (r MsaJob) WriteFasta(path string) error {
 	return nil
 }
 
-func NewMsaJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string) (JobRequest, error) {
+func NewMsaJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := MsaJob{
 		max(strings.Count(query, ">"), 1),
 		dbs,
@@ -52,11 +52,12 @@ func NewMsaJobRequest(query string, dbs []string, validDbs []Params, mode string
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobMsa,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobMsa,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	ids := make([]string, len(validDbs))

backend/pairjob.go

@@ -34,19 +34,20 @@ func (r PairJob) WriteFasta(path string) error {
 	return nil
 }
 
-func NewPairJobRequest(query string, mode string, mail string) (JobRequest, error) {
+func NewPairJobRequest(query string, mode string, mail string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := PairJob{
 		max(strings.Count(query, ">"), 1),
 		mode,
 		query,
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobPair,
-		job,
-		mail,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobPair,
+		Job:       job,
+		Email:     mail,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	return request, nil

backend/searchjob.go

@@ -68,7 +68,7 @@ func isIn(num string, params []string) int {
 
 var validTaxonFilter = regexp.MustCompile(`^[0-9]+(,!?[0-9]+)*$|^$`).MatchString
 
-func NewSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string) (JobRequest, error) {
+func NewSearchJobRequest(query string, dbs []string, validDbs []Params, mode string, resultPath string, email string, taxfilter string, otelTrace OtelTraceContext) (JobRequest, error) {
 	job := SearchJob{
 		max(strings.Count(query, ">"), 1),
 		dbs,
@@ -78,11 +78,12 @@ func NewSearchJobRequest(query string, dbs []string, validDbs []Params, mode str
 	}
 
 	request := JobRequest{
-		job.Hash(),
-		StatusPending,
-		JobSearch,
-		job,
-		email,
+		Id:        job.Hash(),
+		Status:    StatusPending,
+		Type:      JobSearch,
+		Job:       job,
+		Email:     email,
+		OtelTrace: otelTrace.Ptr(),
 	}
 
 	ids := make([]string, len(validDbs))