Can Someone please help us out with Harbor Unknown Blob Issue

[Shashankjain12]

Hi we have deployed harbor using helm Chart and while pushing the images we are facing Unknown Blob issue.
Hereby attaching the relevant installation details related to helm chart

appVersion: 2.13.0
Chart Version: 1.17.0

Values.yml

caSecretName: ""
cache:
  enabled: false
  expireHours: 24
containerSecurityContext:
  allowPrivilegeEscalation: false
  capabilities:
    drop:
    - ALL
  privileged: false
  runAsNonRoot: true
  seccompProfile:
    type: RuntimeDefault
core:
  affinity: {}
  artifactPullAsyncFlushDuration: null
  automountServiceAccountToken: false
  configureUserSettings: null
  existingSecret: ""
  existingXsrfSecret: ""
  existingXsrfSecretKey: CSRF_KEY
  extraEnvVars: []
  gdpr:
    auditLogsCompliant: false
    deleteUser: false
  image:
    repository: goharbor/harbor-core
    tag: v2.13.0
  initContainers: []
  nodeSelector: {}
  podAnnotations:
    newrelic.io/scrape: "true"
    prometheus.io/path: /metrics
    prometheus.io/port: "8001"
    prometheus.io/scrape: "true"
  podLabels: {}
  priorityClassName: null
  quotaUpdateProvider: db
  replicas: 3
  revisionHistoryLimit: 10
  secret: ""
  secretName: ""
  serviceAccountName: ""
  serviceAnnotations: {}
  startupProbe:
    enabled: true
    initialDelaySeconds: 10
  tokenCert: ""
  tokenKey: ""
  tolerations: []
  topologySpreadConstraints: []
  xsrfKey: ""
database:
  external:
    coreDatabase: registry
    existingSecret: ""
    host: postgresql-ha-pgpool
    password: 6b1Zl2bCjB
    port: "5432"
    sslmode: disable
    username: postgres
  internal:
    affinity: {}
    automountServiceAccountToken: false
    extrInitContainers: []
    extraEnvVars: []
    image:
      repository: goharbor/harbor-db
      tag: v2.13.0
    initContainer:
      migrator: {}
      permissions: {}
    livenessProbe:
      timeoutSeconds: 1
    nodeSelector: {}
    password: changeit
    priorityClassName: null
    readinessProbe:
      timeoutSeconds: 1
    serviceAccountName: ""
    shmSizeLimit: 512Mi
    tolerations: []
  maxIdleConns: 100
  maxOpenConns: 900
  podAnnotations: {}
  podLabels: {}
  type: external
enableMigrateHelmHook: false
existingSecretAdminPasswordKey: HARBOR_ADMIN_PASSWORD
existingSecretSecretKey: ""
exporter:
  affinity: {}
  automountServiceAccountToken: false
  cacheCleanInterval: 14400
  cacheDuration: 23
  extraEnvVars: []
  image:
    repository: goharbor/harbor-exporter
    tag: v2.13.0
  nodeSelector: {}
  podAnnotations: {}
  podLabels: {}
  priorityClassName: null
  replicas: 3
  revisionHistoryLimit: 10
  serviceAccountName: ""
  tolerations: []
  topologySpreadConstraints: []
expose:
  clusterIP:
    annotations: {}
    labels: {}
    name: harbor
    ports:
      httpPort: 80
      httpsPort: 443
    staticClusterIP: ""
  ingress:
    annotations:
      ingress.kubernetes.io/proxy-body-size: "0"
      ingress.kubernetes.io/ssl-redirect: "true"
      nginx.ingress.kubernetes.io/proxy-body-size: "0"
      nginx.ingress.kubernetes.io/ssl-redirect: "true"
    className: ""
    controller: default
    hosts:
      core: harbor.omnisphere.ai
    kubeVersionOverride: ""
    labels: {}
  loadBalancer:
    IP: ""
    annotations: {}
    labels: {}
    name: harbor
    ports:
      httpPort: 80
      httpsPort: 443
    sourceRanges: []
  nodePort:
    annotations: {}
    labels: {}
    name: harbor
    ports:
      http:
        nodePort: 30002
        port: 80
      https:
        nodePort: 30003
        port: 443
  tls:
    auto:
      commonName: ""
    certSource: secret
    enabled: false
    secret:
      secretName: omnisphere-crt
  type: clusterIP
externalURL: https://harbor.omnisphere.ai
harborAdminPassword: Harbor12345
imagePullPolicy: IfNotPresent
imagePullSecrets: null
internalTLS:
  certSource: auto
  core:
    crt: ""
    key: ""
    secretName: ""
  enabled: false
  jobservice:
    crt: ""
    key: ""
    secretName: ""
  portal:
    crt: ""
    key: ""
    secretName: ""
  registry:
    crt: ""
    key: ""
    secretName: ""
  strong_ssl_ciphers: false
  trivy:
    crt: ""
    key: ""
    secretName: ""
  trustCa: ""
ipFamily:
  ipv4:
    enabled: true
  ipv6:
    enabled: true
jobservice:
  affinity: {}
  automountServiceAccountToken: false
  existingSecret: ""
  existingSecretKey: JOBSERVICE_SECRET
  extraEnvVars: []
  image:
    repository: goharbor/harbor-jobservice
    tag: v2.13.0
  initContainers: []
  jobLoggers:
  - file
  loggerSweeperDuration: 14
  maxJobWorkers: 10
  nodeSelector: {}
  notification:
    webhook_job_http_client_timeout: 3
    webhook_job_max_retry: 3
  podAnnotations:
    newrelic.io/scrape: "true"
    prometheus.io/path: /metrics
    prometheus.io/port: "8001"
    prometheus.io/scrape: "true"
  podLabels: {}
  priorityClassName: null
  reaper:
    max_dangling_hours: 168
    max_update_hours: 24
  replicas: 3
  revisionHistoryLimit: 10
  secret: ""
  serviceAccountName: ""
  tolerations: []
  topologySpreadConstraints: null
logLevel: info
metrics:
  core:
    path: /metrics
    port: 8001
  enabled: true
  exporter:
    path: /metrics
    port: 8001
  jobservice:
    path: /metrics
    port: 8001
  registry:
    path: /metrics
    port: 8001
  serviceMonitor:
    additionalLabels: {}
    enabled: false
    interval: ""
    metricRelabelings: []
    relabelings: []
nginx:
  affinity: {}
  automountServiceAccountToken: false
  extraEnvVars: []
  image:
    repository: goharbor/nginx-photon
    tag: v2.13.0
  nodeSelector: {}
  podAnnotations: {}
  podLabels: {}
  priorityClassName: null
  replicas: 3
  revisionHistoryLimit: 10
  serviceAccountName: ""
  tolerations: []
  topologySpreadConstraints: []
persistence:
  enabled: true
  imageChartStorage:
    azure:
      accountkey: base64encodedaccountkey
      accountname: accountname
      container: containername
      existingSecret: ""
    disableredirect: false
    filesystem:
      rootdirectory: /storage
    gcs:
      bucket: bucketname
      encodedkey: base64-encoded-json-key-file
      existingSecret: ""
      useWorkloadIdentity: false
    oss:
      accesskeyid: accesskeyid
      accesskeysecret: accesskeysecret
      bucket: bucketname
      existingSecret: ""
      region: regionname
    s3:
      bucket: bucketname
      region: us-west-1
    swift:
      authurl: https://storage.myprovider.com/v3/auth
      container: containername
      existingSecret: ""
      password: password
      username: username
    type: filesystem
  persistentVolumeClaim:
    database:
      accessMode: ReadWriteOnce
      annotations: {}
      existingClaim: ""
      size: 50Gi
      storageClass: ""
      subPath: ""
    jobservice:
      jobLog:
        accessMode: ReadWriteOnce
        annotations: {}
        existingClaim: ""
        size: 100Gi
        storageClass: ""
        subPath: ""
    redis:
      accessMode: ReadWriteOnce
      annotations: {}
      existingClaim: ""
      size: 50Gi
      storageClass: ""
      subPath: ""
    registry:
      accessMode: ReadWriteOnce
      annotations: {}
      existingClaim: ""
      size: 1000Gi
      storageClass: ""
      subPath: ""
    trivy:
      accessMode: ReadWriteOnce
      annotations: {}
      existingClaim: ""
      size: 50Gi
      storageClass: ""
      subPath: ""
  resourcePolicy: keep
portal:
  affinity: {}
  automountServiceAccountToken: false
  extraEnvVars: []
  image:
    repository: goharbor/harbor-portal
    tag: v2.13.0
  initContainers: []
  nodeSelector: {}
  podAnnotations: {}
  podLabels: {}
  priorityClassName: null
  replicas: 3
  revisionHistoryLimit: 10
  serviceAccountName: ""
  serviceAnnotations: {}
  tolerations: []
  topologySpreadConstraints: []
proxy:
  components:
  - core
  - jobservice
  - trivy
  httpProxy: null
  httpsProxy: null
  noProxy: 127.0.0.1,localhost,.local,.internal,0.0.0.0
redis:
  external:
    addr: redis-ha-master:6379
    coreDatabaseIndex: "0"
    existingSecret: ""
    jobserviceDatabaseIndex: "1"
    password: BVybXovdV3
    registryDatabaseIndex: "2"
    sentinelMasterSet: ""
    tlsOptions:
      enable: false
    trivyAdapterIndex: "5"
    username: ""
  internal:
    affinity: {}
    automountServiceAccountToken: false
    extraEnvVars: []
    image:
      repository: goharbor/redis-photon
      tag: v2.13.0
    initContainers: []
    jobserviceDatabaseIndex: "1"
    nodeSelector: {}
    priorityClassName: null
    registryDatabaseIndex: "2"
    serviceAccountName: ""
    tolerations: []
    trivyAdapterIndex: "5"
  podAnnotations: {}
  podLabels: {}
  type: external
registry:
  affinity: {}
  automountServiceAccountToken: false
  controller:
    extraEnvVars: []
    image:
      repository: goharbor/harbor-registryctl
      tag: v2.13.0
  credentials:
    existingSecret: ""
    htpasswdString: ""
    password: harbor_registry_password
    username: harbor_registry_user
  existingSecret: ""
  existingSecretKey: REGISTRY_HTTP_SECRET
  initContainers: []
  middleware:
    cloudFront:
      baseurl: example.cloudfront.net
      duration: 3000s
      ipfilteredby: none
      keypairid: KEYPAIRID
      privateKeySecret: my-secret
    enabled: false
    type: cloudFront
  nodeSelector: {}
  podAnnotations:
    newrelic.io/scrape: "true"
    prometheus.io/path: /metrics
    prometheus.io/port: "8001"
    prometheus.io/scrape: "true"
  podLabels: {}
  priorityClassName: null
  registry:
    extraEnvVars: []
    image:
      repository: goharbor/registry-photon
      tag: v2.13.0
  relativeurls: false
  replicas: 3
  revisionHistoryLimit: 10
  secret: ""
  serviceAccountName: ""
  tolerations: []
  topologySpreadConstraints: []
  upload_purging:
    age: 168h
    dryrun: false
    enabled: true
    interval: 24h
secretKey: not-a-secure-key
trace:
  enabled: false
  jaeger:
    endpoint: http://hostname:14268/api/traces
  otel:
    compression: false
    endpoint: hostname:4318
    insecure: true
    timeout: 10
    url_path: /v1/traces
  provider: jaeger
  sample_rate: 1
trivy:
  affinity: {}
  automountServiceAccountToken: false
  debugMode: false
  enabled: true
  extraEnvVars: []
  gitHubToken: ""
  ignoreUnfixed: false
  image:
    repository: goharbor/trivy-adapter-photon
    tag: v2.13.0
  initContainers: []
  insecure: false
  nodeSelector: {}
  offlineScan: false
  podAnnotations: {}
  podLabels: {}
  priorityClassName: null
  replicas: 3
  resources:
    limits:
      cpu: 1
      memory: 1Gi
    requests:
      cpu: 200m
      memory: 512Mi
  securityCheck: vuln
  serviceAccountName: ""
  severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
  skipJavaDBUpdate: false
  skipUpdate: false
  timeout: 5m0s
  tolerations: []
  topologySpreadConstraints: []
  vulnType: os,library
updateStrategy:
  type: RollingUpdate

Harbor Push error

+ docker push harbor.omnisphere.ai/ai/chess/infinity-chess/backend:1.1.9-stg
The push refers to repository [harbor.omnisphere.ai/ai/chess/infinity-chess/backend]
5bfca4f3e4d2: Preparing
de1127472504: Preparing
f6d6e6d0cc8a: Preparing
69e040df9549: Preparing
f5a88fcbf8c3: Preparing
fefe0c5bbf1c: Preparing
61c5f80ac6bf: Preparing
f6d6e6d0cc8a: Waiting
69e040df9549: Waiting
f5a88fcbf8c3: Waiting
6a01ffdc9090: Preparing
a483da8ab3e9: Preparing
fefe0c5bbf1c: Waiting
5bfca4f3e4d2: Waiting
61c5f80ac6bf: Waiting
a483da8ab3e9: Waiting
6a01ffdc9090: Waiting
de1127472504: Waiting
de1127472504: Layer already exists
f6d6e6d0cc8a: Layer already exists
69e040df9549: Layer already exists
f5a88fcbf8c3: Layer already exists
fefe0c5bbf1c: Layer already exists
61c5f80ac6bf: Layer already exists
6a01ffdc9090: Layer already exists
a483da8ab3e9: Layer already exists
5bfca4f3e4d2: Retrying in 5 seconds
5bfca4f3e4d2: Retrying in 4 seconds
5bfca4f3e4d2: Retrying in 3 seconds
5bfca4f3e4d2: Retrying in 2 seconds
5bfca4f3e4d2: Retrying in 1 second
5bfca4f3e4d2: Retrying in 10 seconds
5bfca4f3e4d2: Retrying in 9 seconds
5bfca4f3e4d2: Retrying in 8 seconds
5bfca4f3e4d2: Retrying in 7 seconds
5bfca4f3e4d2: Retrying in 6 seconds
5bfca4f3e4d2: Retrying in 5 seconds
5bfca4f3e4d2: Retrying in 4 seconds
5bfca4f3e4d2: Retrying in 3 seconds
5bfca4f3e4d2: Retrying in 2 seconds
5bfca4f3e4d2: Retrying in 1 second
unknown blob

Logs of Harbor Registry Pods

 4ms http.response.status=200 http.response.written=4091
192.172.67.145 - - [25/Dec/2025:11:53:48 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/manifests/sha256:835b8c88dba6bdfcf0d0b752c4f807b32dc03e9fc5cf6d88323bfd7fbaf0eb0a HTTP/1.1" 200 4091 "" "containerd/1.7.27"
time="2025-12-25T11:53:49.102533303Z" level=info msg="authorized request" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=9e3330b4-cebe-49d8-96a6-32a69e32b59b http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:aae50742b57eb27f616348b50df453f6144853b9f5400e8fb96792747e49cf59" http.request.useragent="containerd/1.7.27" vars.digest="sha256:aae50742b57eb27f616348b50df453f6144853b9f5400e8fb96792747e49cf59" vars.name="ai/infra/infra-management/admin-portal-website"
time="2025-12-25T11:53:49.155105537Z" level=info msg="response completed" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=9e3330b4-cebe-49d8-96a6-32a69e32b59b http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:aae50742b57eb27f616348b50df453f6144853b9f5400e8fb96792747e49cf59" http.request.useragent="containerd/1.7.27" http.response.contenttype="application/octet-stream" http.response.duration=142.770505ms http.response.status=200 http.response.written=10533
192.172.67.145 - - [25/Dec/2025:11:53:49 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:aae50742b57eb27f616348b50df453f6144853b9f5400e8fb96792747e49cf59 HTTP/1.1" 200 10533 "" "containerd/1.7.27"
time="2025-12-25T11:53:49.264908728Z" level=info msg="authorized request" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=bbde7cfd-f307-474e-976e-51560a7108b5 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:d53345f33f184b0252d05c2c8a8a37adfc01f87cc65f316b3545e9743afd1f19" http.request.useragent="containerd/1.7.27" vars.digest="sha256:d53345f33f184b0252d05c2c8a8a37adfc01f87cc65f316b3545e9743afd1f19" vars.name="ai/infra/infra-management/admin-portal-website"
time="2025-12-25T11:53:49.276256175Z" level=info msg="authorized request" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=60f72313-64d8-46a7-adb0-730acfa1c44f http.request.method=GET http.request.remoteaddr=10.10.25.147 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:6e66b82b2509ebbb6f84ed9c4475218ba5be98b2a932a23f4a7f895e6fefa1c0" http.request.useragent="containerd/1.7.27" vars.digest="sha256:6e66b82b2509ebbb6f84ed9c4475218ba5be98b2a932a23f4a7f895e6fefa1c0" vars.name="ai/infra/infra-management/admin-portal-website"
time="2025-12-25T11:53:49.412220666Z" level=info msg="response completed" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=bbde7cfd-f307-474e-976e-51560a7108b5 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:d53345f33f184b0252d05c2c8a8a37adfc01f87cc65f316b3545e9743afd1f19" http.request.useragent="containerd/1.7.27" http.response.contenttype="application/octet-stream" http.response.duration=235.376859ms http.response.status=200 http.response.written=1078
192.172.67.145 - - [25/Dec/2025:11:53:49 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:d53345f33f184b0252d05c2c8a8a37adfc01f87cc65f316b3545e9743afd1f19 HTTP/1.1" 200 1078 "" "containerd/1.7.27"
time="2025-12-25T11:53:50.002669015Z" level=info msg="authorized request" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=1f966938-d2e8-42a8-9697-e9c357f0e881 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:7e020386ac56c2fc16e4b2b1623d44991f806a9564d48373fff27c3021cc4a64" http.request.useragent="containerd/1.7.27" vars.digest="sha256:7e020386ac56c2fc16e4b2b1623d44991f806a9564d48373fff27c3021cc4a64" vars.name="ai/infra/infra-management/admin-portal-website"
time="2025-12-25T11:53:50.028960247Z" level=info msg="response completed" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=1f966938-d2e8-42a8-9697-e9c357f0e881 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:7e020386ac56c2fc16e4b2b1623d44991f806a9564d48373fff27c3021cc4a64" http.request.useragent="containerd/1.7.27" http.response.contenttype="application/octet-stream" http.response.duration=130.245185ms http.response.status=200 http.response.written=1471
192.172.67.145 - - [25/Dec/2025:11:53:49 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:7e020386ac56c2fc16e4b2b1623d44991f806a9564d48373fff27c3021cc4a64 HTTP/1.1" 200 1471 "" "containerd/1.7.27"
time="2025-12-25T11:53:50.168583012Z" level=info msg="authorized request" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=93614b06-9f6c-4d1f-93ea-edb667a60fa2 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:c5a2cc10a3124a5de81c94f9d44c321a84a2e34e3eb7470b0db8e3faa6af075b" http.request.useragent="containerd/1.7.27" vars.digest="sha256:c5a2cc10a3124a5de81c94f9d44c321a84a2e34e3eb7470b0db8e3faa6af075b" vars.name="ai/infra/infra-management/admin-portal-website"
192.172.67.145 - - [25/Dec/2025:11:53:50 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:c5a2cc10a3124a5de81c94f9d44c321a84a2e34e3eb7470b0db8e3faa6af075b HTTP/1.1" 200 27837247 "" "containerd/1.7.27"
time="2025-12-25T11:53:50.985961Z" level=info msg="response completed" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=93614b06-9f6c-4d1f-93ea-edb667a60fa2 http.request.method=GET http.request.remoteaddr=10.10.25.140 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:c5a2cc10a3124a5de81c94f9d44c321a84a2e34e3eb7470b0db8e3faa6af075b" http.request.useragent="containerd/1.7.27" http.response.contenttype="application/octet-stream" http.response.duration=937.568072ms http.response.status=200 http.response.written=27837247
192.172.19.5 - - [25/Dec/2025:11:53:52 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:53:52 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:53:55 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:53:56 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:02 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:02 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:05 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:06 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:12 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:12 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:15 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:16 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:22 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:22 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:25 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:26 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:32 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:32 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:35 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:36 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:42 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:42 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:45 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:46 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:54:52 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:54:52 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:54:55 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:54:56 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:55:02 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:55:02 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:55:05 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:55:06 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:55:12 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
time="2025-12-25T11:55:12.243842986Z" level=info msg="redis: connect redis-ha-master:6379" go.version=go1.23.8 instance.id=197aa197-e82c-4edc-8a46-42f17066545c redis.connect.duration=2.844097ms service=registry version=v2.8.3-21-gea065fef
192.172.67.145 - - [25/Dec/2025:11:53:49 +0000] "GET /v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:6e66b82b2509ebbb6f84ed9c4475218ba5be98b2a932a23f4a7f895e6fefa1c0 HTTP/1.1" 200 702308702 "" "containerd/1.7.27"
time="2025-12-25T11:55:12.245150368Z" level=info msg="response completed" go.version=go1.23.8 http.request.host=harbor.omnisphere.ai http.request.id=60f72313-64d8-46a7-adb0-730acfa1c44f http.request.method=GET http.request.remoteaddr=10.10.25.147 http.request.uri="/v2/ai/infra/infra-management/admin-portal-website/blobs/sha256:6e66b82b2509ebbb6f84ed9c4475218ba5be98b2a932a23f4a7f895e6fefa1c0" http.request.useragent="containerd/1.7.27" http.response.contenttype="application/octet-stream" http.response.duration=1m23.052236097s http.response.status=200 http.response.written=702308702
10.10.25.12 - - [25/Dec/2025:11:55:12 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:55:15 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:55:16 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:55:22 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:55:22 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:55:25 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:55:26 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
192.172.19.5 - - [25/Dec/2025:11:55:32 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"
10.10.25.12 - - [25/Dec/2025:11:55:32 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
10.10.25.12 - - [25/Dec/2025:11:55:35 +0000] "GET / HTTP/1.1" 200 0 "" "kube-probe/1.32"
192.172.67.145 - - [25/Dec/2025:11:55:36 +0000] "GET / HTTP/1.1" 200 0 "" "Go-http-client/1.1"

Sometime error also comes likes this in registry logs

2025/12/26 05:37:22 http: superfluous response.WriteHeader call from github.com/docker/distribution/vendor/github.com/gorilla/handlers.(*responseLogger).WriteHeader (handlers.go:125)
time="2025-12-26T05:37:22.001065567Z" level=error msg="response completed with error" auth.user.name="harbor_registry_user" err.code=unknown err.detail="client disconnected" err.message="unknown error" go.version=go1.23.8 http.request.contenttype="application/octet-stream" http.request.host=harbor.omnisphere.ai http.request.id=f7736b33-3386-483d-b492-cc15b0c28c90 http.request.method=PATCH http.request.remoteaddr=10.10.25.159 http.request.uri="/v2/ai/omnium/model-training/sql-query-generator/blobs/uploads/c3ed5816-68bb-4017-8429-6f32e7873b82?_state=V2Rr7CZrnPghPZSUTKBbbaM9BFQmfvRDJA0yVS3tUu57Ik5hbWUiOiJhaS9vbW5pdW0vbW9kZWwtdHJhaW5pbmcvc3FsLXF1ZXJ5LWdlbmVyYXRvciIsIlVVSUQiOiJjM2VkNTgxNi02OGJiLTQwMTctODQyOS02ZjMyZTc4NzNiODIiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjUtMTItMjZUMDU6MzI6MTEuODMxNTIyMzY5WiJ9" http.request.useragent="docker/28.3.0 go/go1.24.4 git-commit/265f709 kernel/6.8.0-87-generic os/linux arch/amd64 UpstreamClient(Docker-Client/28.3.0 \(linux\))" http.response.contenttype="application/json; charset=utf-8" http.response.duration=5m9.75736207s http.response.status=500 http.response.written=89 vars.name="ai/omnium/model-training/sql-query-generator" vars.uuid=c3ed5816-68bb-4017-8429-6f32e7873b82
192.172.19.5 - - [26/Dec/2025:05:32:12 +0000] "PATCH /v2/ai/omnium/model-training/sql-query-generator/blobs/uploads/c3ed5816-68bb-4017-8429-6f32e7873b82?_state=V2Rr7CZrnPghPZSUTKBbbaM9BFQmfvRDJA0yVS3tUu57Ik5hbWUiOiJhaS9vbW5pdW0vbW9kZWwtdHJhaW5pbmcvc3FsLXF1ZXJ5LWdlbmVyYXRvciIsIlVVSUQiOiJjM2VkNTgxNi02OGJiLTQwMTctODQyOS02ZjMyZTc4NzNiODIiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjUtMTItMjZUMDU6MzI6MTEuODMxNTIyMzY5WiJ9 HTTP/1.1" 500 89 "" "docker/28.3.0 go/go1.24.4 git-commit/265f709 kernel/6.8.0-87-generic os/linux arch/amd64 UpstreamClient(Docker-Client/28.3.0 \\(linux\\))"

time="2025-12-26T05:50:24.41945825Z" level=error msg="response completed with error" auth.user.name="harbor_registry_user" err.code=unknown err.detail="client disconnected" err.message="unknown error" go.version=go1.23.8 http.request.contenttype="application/octet-stream" http.request.host=harbor.omnisphere.ai http.request.id=9b700ff0-6a25-40c6-af55-116eb922ec87 http.request.method=PATCH http.request.remoteaddr=10.10.25.159 http.request.uri="/v2/ai/omnium/model-training/sql-query-generator/blobs/uploads/7355638c-c8a9-4f68-be93-dfbc50209cf6?_state=ggCUVyS4IoZT61ABg8y-QkVdDTi9ybgctMd0xvN83KJ7Ik5hbWUiOiJhaS9vbW5pdW0vbW9kZWwtdHJhaW5pbmcvc3FsLXF1ZXJ5LWdlbmVyYXRvciIsIlVVSUQiOiI3MzU1NjM4Yy1jOGE5LTRmNjgtYmU5My1kZmJjNTAyMDljZjYiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjUtMTItMjZUMDU6NDg6MTcuMDQ4MjkxOTY0WiJ9" http.request.useragent="docker/28.3.0 go/go1.24.4 git-commit/265f709 kernel/6.8.0-87-generic os/linux arch/amd64 UpstreamClient(Docker-Client/28.3.0 \(linux\))" http.response.contenttype="application/json; charset=utf-8" http.response.duration=2m6.998635642s http.response.status=500 http.response.written=89 vars.name="ai/omnium/model-training/sql-query-generator" vars.uuid=7355638c-c8a9-4f68-be93-dfbc50209cf6

[stonezdj]

close it because there is existing issue #22683