spring webflux, webclient + zipkin

Есть проект на spring boot 3 webflux. К нему нужно подлючить метрики (micrometer) и трасеровку по traceId. Для теста подняли в докере prometheus, grafana

version: '3'

services:
  tempo:
    image: grafana/tempo
    extra_hosts: ['host.docker.internal:host-gateway']
    command: [ "-config.file=/etc/tempo.yaml" ]
    volumes:
      - ./docker/tempo/tempo-local.yaml:/etc/tempo.yaml:ro
      - ./tempo-data:/tmp/tempo
    ports:
      - "14268"  
      - "9411:9411"

  loki:
    image: grafana/loki
    extra_hosts: ['host.docker.internal:host-gateway']
    command: [ "-config.file=/etc/loki/local-config.yaml" ]
    ports:
      - "3100:3100"                                    
    environment:
      - JAEGER_AGENT_HOST=tempo
      - JAEGER_ENDPOINT=http://tempo:14268/api/traces 
      - JAEGER_SAMPLER_TYPE=const
      - JAEGER_SAMPLER_PARAM=1

  prometheus:
    image: prom/prometheus
    extra_hosts: ['host.docker.internal:host-gateway']
    command:
      - --enable-feature=exemplar-storage
      - --config.file=/etc/prometheus/prometheus.yml
    volumes:
      - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana
    extra_hosts: ['host.docker.internal:host-gateway']
    volumes:
      - ./docker/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
      - ./docker/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_AUTH_DISABLE_LOGIN_FORM=true
    ports:
      - "3000:3000"

Зависимости

    implementation 'org.springframework.boot:spring-boot-starter-actuator'
    implementation 'org.springframework.boot:spring-boot-starter-aop'
    implementation 'io.micrometer:micrometer-tracing-bridge-brave'
    implementation 'io.micrometer:micrometer-registry-prometheus:latest.release'
    implementation 'io.zipkin.reporter2:zipkin-reporter-brave'
    implementation 'com.github.loki4j:loki-logback-appender:1.4.0'

    implementation 'io.micrometer:micrometer-observation'
    implementation 'io.micrometer:micrometer-tracing'
    implementation 'io.micrometer:context-propagation:latest.integration'
    implementation 'io.projectreactor:reactor-core:3.5.3'

Метрики снимаются норм, логи тоже отправляются с traceId но проблема с отправкой запросов на http://localhost:9411/api/v2/spans. Выдает ошибку

org.springframework.web.reactive.function.client.WebClientRequestException: Failed to resolve 'localhost' [A(1)] after 2 queries 
    at org.springframework.web.reactive.function.client.ExchangeFunctions$DefaultExchangeFunction.lambda$wrapException$9(ExchangeFunctions.java:136)
    Suppressed: reactor.core.publisher.FluxOnAssembly$OnAssemblyException: 
Error has been observed at the following site(s):
    *__checkpoint ⇢ Request to POST http://localhost:9411/api/v2/spans [DefaultWebClient]
Original Stack Trace:
        at org.springframework.web.reactive.function.client.ExchangeFunctions$DefaultExchangeFunction.lambda$wrapException$9(ExchangeFunctions.java:136)
        at reactor.core.publisher.MonoErrorSupplied.subscribe(MonoErrorSupplied.java:55)
        at reactor.core.publisher.Mono.subscribe(Mono.java:4485)
        at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:103)
        at reactor.core.publisher.FluxPeek$PeekSubscriber.onError(FluxPeek.java:222)
        at reactor.core.publisher.FluxPeek$PeekSubscriber.onError(FluxPeek.java:222)
        at reactor.core.publisher.FluxPeek$PeekSubscriber.onError(FluxPeek.java:222)
        at reactor.core.publisher.MonoNext$NextSubscriber.onError(MonoNext.java:93)
        at reactor.core.publisher.MonoFlatMapMany$FlatMapManyMain.onError(MonoFlatMapMany.java:204)
        at reactor.core.publisher.SerializedSubscriber.onError(SerializedSubscriber.java:124)
        at reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.whenError(FluxRetryWhen.java:225)
        at reactor.core.publisher.FluxRetryWhen$RetryWhenOtherSubscriber.onError(FluxRetryWhen.java:274)
        at reactor.core.publisher.FluxContextWriteRestoringThreadLocals$ContextWriteRestoringThreadLocalsSubscriber.onError(FluxContextWriteRestoringThreadLocals.java:139)
        at reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.maybeOnError(FluxConcatMapNoPrefetch.java:326)
        at reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.onNext(FluxConcatMapNoPrefetch.java:211)
        at reactor.core.publisher.FluxContextWriteRestoringThreadLocals$ContextWriteRestoringThreadLocalsSubscriber.onNext(FluxContextWriteRestoringThreadLocals.java:117)
        at reactor.core.publisher.SinkManyEmitterProcessor.drain(SinkManyEmitterProcessor.java:471)
        at reactor.core.publisher.SinkManyEmitterProcessor$EmitterInner.drainParent(SinkManyEmitterProcessor.java:615)
        at reactor.core.publisher.FluxPublish$PubSubInner.request(FluxPublish.java:602)
        at reactor.core.publisher.FluxContextWriteRestoringThreadLocals$ContextWriteRestoringThreadLocalsSubscriber.request(FluxContextWriteRestoringThreadLocals.java:162)
        at reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.request(FluxConcatMapNoPrefetch.java:336)
        at reactor.core.publisher.FluxContextWriteRestoringThreadLocals$ContextWriteRestoringThreadLocalsSubscriber.request(FluxContextWriteRestoringThreadLocals.java:162)
        at reactor.core.publisher.Operators$DeferredSubscription.request(Operators.java:1717)
        at reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onError(FluxRetryWhen.java:192)
        at reactor.core.publisher.MonoCreate$DefaultMonoSink.error(MonoCreate.java:201)
        at reactor.netty.http.client.HttpClientConnect$MonoHttpConnect$ClientTransportSubscriber.onError(HttpClientConnect.java:311)
        at reactor.core.publisher.MonoCreate$DefaultMonoSink.error(MonoCreate.java:201)
        at reactor.netty.resources.DefaultPooledConnectionProvider$DisposableAcquire.onError(DefaultPooledConnectionProvider.java:162)
        at reactor.netty.internal.shaded.reactor.pool.AbstractPool$Borrower.fail(AbstractPool.java:475)
        at reactor.netty.internal.shaded.reactor.pool.SimpleDequePool.lambda$drainLoop$9(SimpleDequePool.java:429)
        at reactor.core.publisher.FluxDoOnEach$DoOnEachSubscriber.onError(FluxDoOnEach.java:186)
        at reactor.core.publisher.MonoCreate$DefaultMonoSink.error(MonoCreate.java:201)
        at reactor.netty.resources.DefaultPooledConnectionProvider$PooledConnectionAllocator$PooledConnectionInitializer.onError(DefaultPooledConnectionProvider.java:560)
        at reactor.core.publisher.MonoFlatMap$FlatMapMain.secondError(MonoFlatMap.java:241)
        at reactor.core.publisher.MonoFlatMap$FlatMapInner.onError(MonoFlatMap.java:315)
        at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:106)
        at reactor.core.publisher.Operators.error(Operators.java:198)
        at reactor.core.publisher.MonoError.subscribe(MonoError.java:53)
        at reactor.core.publisher.Mono.subscribe(Mono.java:4485)
        at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:103)
        at reactor.netty.transport.TransportConnector$MonoChannelPromise.tryFailure(TransportConnector.java:587)
        at reactor.netty.transport.TransportConnector.lambda$doResolveAndConnect$11(TransportConnector.java:394)
        at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
        at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
        at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
        at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
        at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
        at io.netty.util.concurrent.DefaultPromise.setFailure(DefaultPromise.java:110)
        at io.netty.resolver.InetSocketAddressResolver$2.operationComplete(InetSocketAddressResolver.java:86)
        at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
        at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
        at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
        at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
        at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
        at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
        at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
        at io.netty.resolver.dns.DnsResolveContext.finishResolve(DnsResolveContext.java:1096)
        at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1035)
        at io.netty.resolver.dns.DnsResolveContext.query(DnsResolveContext.java:422)
        at io.netty.resolver.dns.DnsResolveContext.access$700(DnsResolveContext.java:66)
        at io.netty.resolver.dns.DnsResolveContext$2.operationComplete(DnsResolveContext.java:493)
        at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
        at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
        at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
        at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
        at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
        at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
        at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
        at io.netty.resolver.dns.DnsQueryContext.tryFailure(DnsQueryContext.java:261)
        at io.netty.resolver.dns.DnsQueryContext$4.run(DnsQueryContext.java:208)
        at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98)
        at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:153)
        at io.netty.util.concurrent.AbstractEventExecutor.runTask$$$capture(AbstractEventExecutor.java:174)
        at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java)
        at io.netty.util.concurrent.AbstractEventExecutor.safeExecute$$$capture(AbstractEventExecutor.java:167)
        at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java)
        at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:406)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.base/java.lang.Thread.run(Thread.java:1589)
    Suppressed: java.lang.Exception: #block terminated with an error
        at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:99)
        at reactor.core.publisher.Mono.block(Mono.java:1710)
        at org.springframework.boot.actuate.autoconfigure.tracing.zipkin.ZipkinWebClientSender$WebClientHttpPostCall.doExecute(ZipkinWebClientSender.java:67)
        at org.springframework.boot.actuate.autoconfigure.tracing.zipkin.ZipkinWebClientSender$WebClientHttpPostCall.doExecute(ZipkinWebClientSender.java:48)
        at zipkin2.Call$Base.execute(Call.java:391)
        at zipkin2.reporter.AsyncReporter$BoundedAsyncReporter.flush(AsyncReporter.java:299)
        at zipkin2.reporter.AsyncReporter$Flusher.run(AsyncReporter.java:378)
        at java.base/java.lang.Thread.run(Thread.java:1589)
Caused by: java.net.UnknownHostException: Failed to resolve 'localhost' [A(1)] after 2 queries 
    at io.netty.resolver.dns.DnsResolveContext.finishResolve(DnsResolveContext.java:1088)
    at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1035)
    at io.netty.resolver.dns.DnsResolveContext.query(DnsResolveContext.java:422)
    at io.netty.resolver.dns.DnsResolveContext.access$700(DnsResolveContext.java:66)
    at io.netty.resolver.dns.DnsResolveContext$2.operationComplete(DnsResolveContext.java:493)
    at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
    at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
    at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
    at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
    at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
    at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
    at io.netty.resolver.dns.DnsQueryContext.tryFailure(DnsQueryContext.java:261)
    at io.netty.resolver.dns.DnsQueryContext$4.run(DnsQueryContext.java:208)
    at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98)
    at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:153)
    at io.netty.util.concurrent.AbstractEventExecutor.runTask$$$capture(AbstractEventExecutor.java:174)
    at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java)
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute$$$capture(AbstractEventExecutor.java:167)
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java)
    at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
    at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:406)
    at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
    at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.base/java.lang.Thread.run(Thread.java:1589)
Caused by: io.netty.resolver.dns.DnsNameResolverTimeoutException: [21405: /[0:0:0:0:0:0:0:1]:53] DefaultDnsQuestion(localhost. IN A) query '21405' via UDP timed out after 5000 milliseconds (no stack trace available)

В связи с этим даные не отправляются и построить трасировку не получается.

Также пробовали отдельно поставить openzipkin/zipkin и отпралять на него, но также вылетает ошибка.

Тут вроде проблема имено с webclient, так как если не использовать webFlux то все отрабатывает.

Кто нибуть сталкивался с данной проблемой?


Ответы (0 шт):