gRPC 메타데이터를 넘어도 traceId가 이어지는 Node.js 분산 트레이싱

// tracing.ts
import { NodeSDK } from '@opentelemetry/sdk-node';
import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { Resource } from '@opentelemetry/resources';
 
const sdk = new NodeSDK({
  resource: new Resource({
    'service.name': 'order-service',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://localhost:4317',
  }),
  instrumentations: [
    new GrpcInstrumentation(),
  ],
});
 
sdk.start();
 
process.on('SIGTERM', () => {
  sdk.shutdown().finally(() => process.exit(0));
});

// index.ts — tracing.ts를 가장 먼저
import './tracing';
import express from 'express';
// ...

// otel-client-interceptor.ts
import { context, propagation } from '@opentelemetry/api';
import * as grpc from '@grpc/grpc-js';
 
export function otelClientInterceptor(
  options: grpc.CallOptions,
  nextCall: (options: grpc.CallOptions) => grpc.InterceptingCall,
): grpc.InterceptingCall {
  return new grpc.InterceptingCall(nextCall(options), {
    start(
      metadata: grpc.Metadata,
      listener: grpc.InterceptingListener,
      next: (metadata: grpc.Metadata, listener: grpc.InterceptingListener) => void,
    ) {
      propagation.inject(context.active(), metadata, {
        set(carrier: grpc.Metadata, key: string, value: string) {
          carrier.set(key, value);
        },
      });
      next(metadata, listener);
    },
  });
}
 
const client = new OrderServiceClient(
  'order-service:50051',
  grpc.credentials.createInsecure(),
  { interceptors: [otelClientInterceptor] },
);

// order-service-handler.ts
import {
  context,
  propagation,
  ROOT_CONTEXT,
  SpanStatusCode,
  trace,
} from '@opentelemetry/api';
import * as grpc from '@grpc/grpc-js';
 
const metadataGetter = {
  get(carrier: grpc.Metadata, key: string) {
    const values = carrier.get(key);
    return values.length > 0 ? String(values[0]) : undefined;
  },
  keys(carrier: grpc.Metadata) {
    return Object.keys(carrier.getMap());
  },
};
 
async function handleCreateOrder(
  call: grpc.ServerUnaryCall<CreateOrderRequest, CreateOrderResponse>,
  callback: grpc.sendUnaryData<CreateOrderResponse>,
) {
  const parentCtx = propagation.extract(ROOT_CONTEXT, call.metadata, metadataGetter);
 
  await context.with(parentCtx, async () => {
    const tracer = trace.getTracer('order-service');
    const span = tracer.startSpan('handle-create-order');
    try {
      const response = await processOrder(call.request);
      span.end();
      callback(null, response);
    } catch (err) {
      span.recordException(err as Error);
      span.setStatus({ code: SpanStatusCode.ERROR });
      span.end();
      callback(err as grpc.ServiceError);
    }
  });
}

import { context, propagation } from '@opentelemetry/api';
 
async function handleHttpRequest(req: Request) {
  const baggage = propagation.createBaggage({
    'user.id': { value: req.user.id },
    'tenant.id': { value: req.tenant.id },
  });
 
  const ctxWithBaggage = propagation.setBaggage(context.active(), baggage);
 
  await context.with(ctxWithBaggage, async () => {
    await orderServiceClient.createOrder(request);
  });
}
 
function getCurrentTenantId(): string | undefined {
  const baggage = propagation.getBaggage(context.active());
  return baggage?.getEntry('tenant.id')?.value;
}

# collector-sidecar.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
 
processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
  k8sattributes:
    extract:
      metadata:
        - k8s.pod.name
        - k8s.namespace.name
        - k8s.node.name
 
exporters:
  otlp:
    endpoint: gateway-collector:4317
    tls:
      insecure: true
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, k8sattributes]
      exporters: [otlp]

# collector-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector-k8sattributes
rules:
  - apiGroups: [""]
    resources: [pods, namespaces]
    verbs: [get, watch, list]
  - apiGroups: [apps]
    resources: [replicasets]
    verbs: [get, watch, list]
  - apiGroups: [extensions]
    resources: [replicasets]
    verbs: [get, watch, list]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector-k8sattributes
subjects:
  - kind: ServiceAccount
    name: otel-collector
    namespace: observability
roleRef:
  kind: ClusterRole
  name: otel-collector-k8sattributes
  apiGroup: rbac.authorization.k8s.io

# gateway-collector.yaml (핵심 부분)
processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
  k8sattributes:
    extract:
      metadata:
        - k8s.pod.name
        - k8s.namespace.name
        - k8s.node.name
  tail_sampling:
    decision_wait: 30s
    num_traces: 50000
    policies:
      - name: errors-policy
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-traces-policy
        type: latency
        latency:
          threshold_ms: 500
      - name: normal-traces-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [k8sattributes, tail_sampling, batch]
      exporters: [otlp]