Just about every developer at some stage needs to see the raw-request/response stream to “know” the correct payload is being sent or to observe the raw response exception.
gcloud cli provides the (--log-http) switch for just such a scenario.
How would cloud SDK users log the request/response?
This section covers some flags and configurations you can enable to see the trace logs for both http-based clients (GCS) and gRPC-based clients (PubSub)
For the most part, gRPC clients that are ultimately based in C will honor the standard debug gRPC Environment Variables:
export GRPC_VERBOSITY=DEBUG
export GRPC_TRACE=all
A notible exception is golang which has some of its own variables as shown in below.
Note, there are much more low-level and comprehensive ways to trace everything.
For example, you can (in golang atleast) decrypt the gRPC traffic automatically with SSLKKEYLOG files and wireshark
or thorough an intercepting proxy.
However, its a fair amount of work to get that setup so this section covers easier, environment-variable based changes.
set HTTPConnection.debuglevel for HTTP clients and the gRPC environment variables
as in:
project='your_project_id'
from http.client import HTTPConnection # py3
# https://docs.python.org/3/library/http.client.html
HTTPConnection.debuglevel = 1
from google.cloud import storage
client = storage.Client(project=project)
for b in client.list_buckets():
print(b.name)
# export GRPC_VERBOSITY=DEBUG GRPC_TRACE=all
from google.cloud import pubsub_v1
publisher = pubsub_v1.PublisherClient()
project_path = f"projects/{project}"
for topic in publisher.list_topics(request={"project": project_path}):
print(topic.name)
for gRPC based clients:
export GRPC_GO_LOG_VERBOSITY_LEVEL=99
export GRPC_GO_LOG_SEVERITY_LEVEL=info
for httpClients, you can wrap the client through http.RoundTripper and use the net/http flags
as in:
package main
import (
"fmt"
"log"
"net/http"
pubsub "cloud.google.com/go/pubsub"
storage "cloud.google.com/go/storage"
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)
const (
projectID = "your_project_id"
)
// https://github.com/googleapis/google-cloud-go/issues/487
// https://github.com/grpc/grpc/blob/master/doc/environment_variables.md
// export GRPC_GO_LOG_SEVERITY_LEVEL=info
// export GRPC_GO_LOG_VERBOSITY_LEVEL=99
// https://pkg.go.dev/net/http
// GODEBUG=http2client=0 # disable HTTP/2 client support
// GODEBUG=http2server=0 # disable HTTP/2 server support
// GODEBUG=http2debug=1 # enable verbose HTTP/2 debug logs
// GODEBUG=http2debug=2 # ... even more verbose, with frame dumps
// export GODEBUG=http2debug=2
type wrapped struct {
base http.RoundTripper
}
func (w wrapped) RoundTrip(r *http.Request) (*http.Response, error) {
fmt.Println("=======================================")
fmt.Println("[GCS_REQUEST]")
fmt.Printf("%v %v\n", r.Method, r.URL.String())
for h, values := range r.Header {
for _, v := range values {
fmt.Printf("%v: %v\n", h, v)
}
}
resp, err := w.base.RoundTrip(r)
fmt.Println("[GCS_RESPONSE]")
if err != nil {
fmt.Printf("GCS_ERROR\n: %v", err)
return resp, err
}
for h, values := range resp.Header {
for _, v := range values {
fmt.Printf("%v: %v\n", h, v)
}
}
return resp, err
}
func main() {
ctx := context.Background()
hc, err := google.DefaultClient(ctx)
if err != nil {
log.Fatalf("%v", err)
}
hc.Transport = wrapped{hc.Transport}
storageClient, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
//storageClient, err := storage.NewClient(ctx)
if err != nil {
fmt.Printf("storage.NewClient: %v", err)
return
}
defer storageClient.Close()
it := storageClient.Buckets(ctx, projectID)
for {
battrs, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
fmt.Printf("storage.Iterating error: %v", err)
return
}
fmt.Printf("Bucket Name: %s\n", battrs.Name)
}
// export GRPC_VERBOSITY=DEBUG
// export GRPC_TRACE=all
// export GRPC_GO_LOG_VERBOSITY_LEVEL=99
// export GRPC_GO_LOG_SEVERITY_LEVEL=info
pubsubClient, err := pubsub.NewClient(ctx, projectID)
if err != nil {
fmt.Printf("pubsub.NewClient: %v", err)
return
}
defer pubsubClient.Close()
pit := pubsubClient.Topics(ctx)
for {
topic, err := pit.Next()
if err == iterator.Done {
break
}
if err != nil {
fmt.Printf("pubssub.Iterating error: %v", err)
return
}
fmt.Printf("Topic Name: %s\n", topic.ID())
}
}
Use Logger with io.grpc and com.google.api.client
as in:
package com.test;
import com.google.cloud.pubsub.v1.TopicAdminClient;
import com.google.cloud.pubsub.v1.TopicAdminClient.ListTopicsPagedResponse;
import com.google.cloud.pubsub.v1.TopicAdminSettings;
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import com.google.pubsub.v1.ListTopicsRequest;
import com.google.pubsub.v1.ProjectName;
import com.google.pubsub.v1.Topic;
import java.util.logging.ConsoleHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
public class TestApp {
public static void main(String[] args) {
TestApp tc = new TestApp();
}
public TestApp() {
try {
ConsoleHandler consoleHandler = new ConsoleHandler();
consoleHandler.setLevel(Level.ALL);
consoleHandler.setFormatter(new SimpleFormatter());
Logger logger = Logger.getLogger("com.google.api.client");
logger.setLevel(Level.ALL);
logger.addHandler(consoleHandler);
Logger gl = Logger.getLogger("io.grpc");
gl.setLevel(Level.FINE);
gl.addHandler(consoleHandler);
Storage storage_service = StorageOptions.newBuilder().build().getService();
for (Bucket b : storage_service.list().iterateAll()) {
System.out.println(b);
}
TopicAdminClient topicClient = TopicAdminClient.create(TopicAdminSettings.newBuilder().build());
ListTopicsRequest listTopicsRequest = ListTopicsRequest.newBuilder()
.setProject(ProjectName.format("your_project_id"))
.build();
ListTopicsPagedResponse response = topicClient.listTopics(listTopicsRequest);
Iterable<Topic> topics = response.iterateAll();
for (Topic topic : topics)
System.out.println(topic);
} catch (Exception ex) {
System.out.println("Error: " + ex);
}
}
}
# for http
export DEBUG=retry-request
export NODE_DEBUG=http
# for gRPC
export GRPC_VERBOSITY=DEBUG
export GRPC_TRACE=all
as in:
var log4js = require("log4js");
var logger = log4js.getLogger();
var { GoogleAuth } = require('google-auth-library');
const {PubSub} = require('@google-cloud/pubsub');
const {Storage} = require('@google-cloud/storage');
async function main() {
// export DEBUG=retry-request
// export NODE_DEBUG=http
var gcs = new Storage();
const [buckets] = await gcs.getBuckets();
buckets.forEach(bucket => logger.info(bucket.name));
// export GRPC_VERBOSITY=DEBUG
// export GRPC_TRACE=all
const pubSubClient = new PubSub({
projectId: 'your_project_id'
});
const [topics] = await pubSubClient.getTopics();
topics.forEach(topic => logger.info(topic.name));
}
main().catch(console.error);
For gRPC use the standard environment variable
for http its still a TODO
http Uses System.Net.Http; but i don’t know how to enable logging via env-var or config file
using System;
using Google.Cloud.Storage.V1;
using Google.Cloud.PubSub.V1;
using System.Threading.Tasks;
using Google.Api.Gax.ResourceNames;
using Google.Api.Gax.Grpc;
namespace main
{
class Program
{
const string projectID = "your_project_id";
[STAThread]
static void Main(string[] args)
{
new Program().Run().Wait();
}
private Task Run()
{
var client = StorageClient.Create();
// TODO
// Uses System.Net.Http; but i don't know how to enable logging via env-var or config file
// https://github.com/googleapis/google-api-dotnet-client/blob/main/Src/Support/Google.Apis.Core/Http/HttpClientFactory.cs#L18
foreach (var b in client.ListBuckets(projectID))
Console.WriteLine(b.Name);
// export GRPC_VERBOSITY=DEBUG
// export GRPC_TRACE=all
PublisherServiceApiClient publisher = PublisherServiceApiClient.Create();
ProjectName projectName = ProjectName.FromProject(projectID);
foreach (Topic t in publisher.ListTopics(projectName))
Console.WriteLine(t.MessageStoragePolicy);
return Task.CompletedTask;
}
}
}
This site supports webmentions. Send me a mention via this form.