⏳
Loading cheatsheet...
GCP deployment workflows, CI/CD and cloud operations practices for production systems.
# ── Google Compute Engine (GCE) ──
# Create a VM instance
gcloud compute instances create my-app-server \
--machine-type e2-medium \
--image-family debian-12 \
--image-project debian-cloud \
--zone us-central1-a \
--boot-disk-size 50GB \
--boot-disk-type pd-ssd \
--tags web-server \
--metadata startup-script='#!/bin/bash
apt-get update && apt-get install -y nginx
systemctl start nginx' \
--network-interface subnetwork=my-subnet \
--service-account my-app-sa@project.iam.gserviceaccount.com
# List instances
gcloud compute instances list --format='table(name,zone,machineType,status)'
# SSH into instance
gcloud compute ssh my-app-server --zone us-central1-a
# Stop / Start / Delete
gcloud compute instances stop my-app-server --zone us-central1-a
gcloud compute instances start my-app-server --zone us-central1-a
gcloud compute instances delete my-app-server --zone us-central1-a
# Create instance template (for MIG)
gcloud compute instance-templates create my-app-template \
--machine-type e2-medium \
--image-family debian-12 \
--image-project debian-cloud \
--boot-disk-size 50GB \
--tags web-server
# Create Managed Instance Group (MIG)
gcloud compute instance-groups managed create my-app-mig \
--base-instance-name my-app \
--size 3 \
--template my-app-template \
--zone us-central1-a
# Auto-scaling
gcloud compute instance-groups managed set-autoscaling my-app-mig \
--zone us-central1-a \
--min-num-replicas 2 \
--max-num-replicas 10 \
--target-cpu-utilization 0.7# ── Google Kubernetes Engine (GKE) ──
# Create GKE Autopilot cluster (serverless, GKE manages nodes)
gcloud container clusters create-auto my-autopilot-cluster \
--region us-central1 \
--network my-vpc \
--subnetwork my-subnet
# Create GKE Standard cluster
gcloud container clusters create my-standard-cluster \
--region us-central1 \
--num-nodes 3 \
--machine-type e2-medium \
--enable-autoscaling \
--min-nodes 1 \
--max-nodes 10 \
--enable-autorepair \
--enable-autoupgrade \
--workload-pool PROJECT_ID.svc.id.goog
# Get credentials
gcloud container clusters get-credentials my-autopilot-cluster \
--region us-central1
# Deploy an application
kubectl create deployment myapp --image=gcr.io/PROJECT_ID/myapp:latest
kubectl expose deployment myapp --port=80 --type=LoadBalancer
kubectl scale deployment myapp --replicas=5
kubectl autoscale deployment myapp --cpu-percent=50 --min=2 --max=10
# Apply manifests
kubectl apply -f k8s/ -R
# View resources
kubectl get all,ing,configmap,secret,pvc -n production
kubectl describe pod myapp-xxx -n production
kubectl logs myapp-xxx -n production --tail=100 -f# ── Cloud Run (Serverless Containers) ──
# Deploy a container image
gcloud run deploy my-service \
--image gcr.io/PROJECT_ID/myapp:latest \
--region us-central1 \
--platform managed \
--allow-unauthenticated \
--cpu 1 \
--memory 512Mi \
--min-instances 1 \
--max-instances 100 \
--concurrency 80 \
--timeout 300 \
--set-env-vars NODE_ENV=production \
--set-secrets DB_PASSWORD=my-db-secret:latest \
--vpc-connector my-vpc-connector \
--service-account my-cloud-run-sa
# Update an existing service
gcloud run services update my-service \
--image gcr.io/PROJECT_ID/myapp:v2.0 \
--region us-central1
# Set IAM (unauthenticated access)
gcloud run services add-iam-policy-binding my-service \
--region us-central1 \
--member allUsers \
--role roles/run.invoker
# List services
gcloud run services list --region us-central1
# View logs
gcloud logging read "resource.type=cloud_run_revision \
AND resource.labels.service_name=my-service" --limit 50
# ── Cloud Run Jobs (batch workloads) ──
gcloud run jobs create my-batch-job \
--image gcr.io/PROJECT_ID/batch-processor \
--tasks 5 \
--parallelism 2 \
--region us-central1
# Execute the job
gcloud run jobs execute my-batch-job --region us-central1| Service | Use Case | Scaling | Management |
|---|---|---|---|
| GCE (VMs) | Full control, legacy apps | Manual or MIG auto-scale | You manage OS, patches |
| GKE Standard | Container orchestration | HPA/VPA/CA auto-scale | You manage node config |
| GKE Autopilot | K8s without node ops | Fully automatic | Google manages everything |
| Cloud Run | Serverless containers | 0→N (request-driven) | No infra management |
| Cloud Functions | Event-driven functions | 0→N (event-driven) | Just code + triggers |
| App Engine | Platform (web apps) | Automatic | Framework-based (Python/Java/Go) |
| Family | Prefix | Best For |
|---|---|---|
| General Purpose | e2, n2, n2d | Web servers, dev environments |
| Memory Optimized | m1, m2 | In-memory databases, analytics |
| Compute Optimized | c2, c2d | CPU-intensive apps, batch |
| GPU / AI | a2, g2 | ML training, inference, rendering |
| Spot / Preemptible | Any + --preemptible | Fault-tolerant, batch jobs (cheap) |
# ── Google Cloud Storage (GCS) ──
# Create bucket (multi-region for high availability)
gsutil mb -l US -c STANDARD gs://my-app-bucket
# Create bucket with versioning & retention
gsutil mb -l US-EAST1 -c STANDARD gs://my-secure-bucket
gsutil versioning set on gs://my-secure-bucket
gsutil retention set 30d gs://my-secure-bucket
# Upload / Download
gsutil cp ./local-file.txt gs://my-app-bucket/data/
gsutil cp -r ./dist/ gs://my-app-bucket/static/
gsutil cp gs://my-app-bucket/data/file.txt ./local/
# Sync (mirror directory)
gsutil rsync -r -d ./local-dir/ gs://my-app-bucket/data/
# Set storage class
gsutil storage class set NEARLINE gs://my-app-bucket/data/*
gsutil storage class set ARCHIVE gs://my-app-bucket/old-data/*
# Lifecycle configuration
cat > lifecycle.json << 'EOF'
{
"lifecycle": {
"rule": [
{
"action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},
"condition": {"age": 30, "matchesPrefix": ["logs/"]}
},
{
"action": {"type": "Delete"},
"condition": {"age": 365, "matchesPrefix": ["temp/"]}
}
]
}
}
EOF
gsutil lifecycle set lifecycle.json gs://my-app-bucket
# Signed URLs (for secure temporary access)
gsutil signurl -d 1h ./keyfile.json gs://my-app-bucket/secret.pdf
# Object metadata
gsutil stat gs://my-app-bucket/data/file.txt-- ── BigQuery: Analytics Data Warehouse ──
-- Create dataset
CREATE SCHEMA my_project.analytics
OPTIONS (
location = 'US',
description = 'Application analytics'
);
-- Create partitioned table
CREATE TABLE my_project.analytics.events (
event_id STRING NOT NULL,
user_id STRING,
event_type STRING,
payload JSON,
created_at TIMESTAMP NOT NULL
)
PARTITION BY DATE(created_at)
CLUSTER BY user_id, event_type
OPTIONS (
partition_expiration_days = 365,
description = 'User events'
);
-- Load data from GCS
LOAD DATA INTO my_project.analytics.events
FROM FILES (
format = 'PARQUET',
uris = ['gs://my-bucket/events/*.parquet']
);
-- Insert streaming data
INSERT INTO my_project.analytics.events (event_id, user_id, event_type, created_at)
VALUES ('e123', 'u456', 'page_view', CURRENT_TIMESTAMP());
-- Query with performance tips
SELECT
user_id,
event_type,
COUNT(*) as event_count,
TIMESTAMP_TRUNC(created_at, HOUR) as event_hour
FROM my_project.analytics.events
WHERE DATE(created_at) >= DATE_SUB(CURRENT_DATE(), INTERVAL 7 DAY)
GROUP BY user_id, event_type, event_hour
ORDER BY event_count DESC
LIMIT 1000;
-- Useful functions
SELECT
SAFE_DIVIDE(success_count, total_count) as success_rate,
ARRAY_AGG(DISTINCT event_type) as event_types,
STRUCT(city, country) as location
FROM ...| Class | Availability | Min Duration | Use Case |
|---|---|---|---|
| Standard | 99.95% (multi-region) | None | Frequently accessed data |
| Nearline | 99.95% | 30 days | Data accessed <1x/month |
| Coldline | 99.95% | 90 days | Data accessed <1x/quarter |
| Archive | 99.95% | 365 days | Long-term backup, compliance |
| Autoclass | Auto-tiers | None | Unknown access patterns |
| AWS S3 | GCS | Operation |
|---|---|---|
| aws s3 mb | gsutil mb | Create bucket |
| aws s3 cp | gsutil cp | Upload/download |
| aws s3 ls | gsutil ls | List objects |
| aws s3 rm | gsutil rm | Delete objects |
| aws s3 sync | gsutil rsync | Sync directories |
| aws s3api presign | gsutil signurl | Signed URL |
| aws s3api get-object | gsutil cat | Read object content |
PARTITION BY and CLUSTER BY to reduce scan cost. Preview queries with LIMIT before running on full data.# ── Cloud SQL (Managed PostgreSQL/MySQL) ──
# Create PostgreSQL instance
gcloud sql instances create my-app-db \
--database-version POSTGRES_16 \
--tier db-custom-2-8192 \
--region us-central1 \
--availability-type REGIONAL \
--storage-type SSD \
--storage-size 100GB \
--storage-auto-increase \
--backup-start-time 02:00 \
--enable-point-in-time-recovery \
--maintenance-window-day=SUN \
--maintenance-window-hour=03 \
--database-flags log_checkpoints=on,log_connections=on \
--network my-vpc
# Create database and user
gcloud sql databases create appdb --instance my-app-db
gcloud sql users create appuser --instance my-app-db \
--password '$(openssl rand -base64 24)'
# Connect (via Cloud SQL Auth Proxy)
cloud-sql-proxy my-project:us-central1:my-app-db
# Set up high availability replica
gcloud sql instances create my-app-db-replica \
--master-instance-name my-app-db \
--region us-west1
# Create read replica (same region)
gcloud sql instances create my-app-db-read \
--master-instance-name my-app-db \
--region us-central1
# Import / Export
gcloud sql export sql my-app-db gs://my-bucket/backups/db.sql \
--database appdb
gcloud sql import sql my-app-db gs://my-bucket/backups/db.sql \
--database appdb# ── Cloud Spanner (Globally distributed relational DB) ──
# Create instance
gcloud spanner instances create my-spanner \
--config=regional-us-central1 \
--nodes=1 \
--display-name="My Spanner Instance"
# Create database
gcloud spanner databases create mydb --instance my-spanner
# Create schema (DDL)
gcloud spanner databases ddl update mydb --instance my-spanner \
--ddl='
CREATE TABLE users (
user_id STRING(36) NOT NULL,
email STRING(255) NOT NULL,
name STRING(255),
created_at TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true),
) PRIMARY KEY (user_id);
CREATE TABLE orders (
order_id STRING(36) NOT NULL,
user_id STRING(36) NOT NULL,
total_amount NUMERIC,
status STRING(50),
created_at TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true),
) PRIMARY KEY (order_id, created_at),
INTERLEAVE IN PARENT users ON DELETE CASCADE;
CREATE INDEX orders_by_user ON orders (user_id, created_at);
'
# Execute query
gcloud spanner databases execute-sql mydb --instance my-spanner \
--sql='SELECT * FROM users WHERE email = @email' \
--params='email="alice@example.com"'// ── Cloud Firestore (NoSQL Document Database) ──
import { initializeApp } from 'firebase/app';
import { getFirestore, collection, doc, setDoc, getDoc, query, where, getDocs, updateDoc, deleteDoc, writeBatch } from 'firebase/firestore';
const db = getFirestore(initializeApp({ projectId: 'my-project' }));
// Add a document
await setDoc(doc(db, 'users', 'alice'), {
name: 'Alice Johnson',
email: 'alice@example.com',
role: 'admin',
createdAt: new Date(),
});
// Get a document
const snap = await getDoc(doc(db, 'users', 'alice'));
if (snap.exists()) console.log(snap.data());
// Query with where clause
const q = query(collection(db, 'users'), where('role', '==', 'admin'));
const users = await getDocs(q);
users.forEach(doc => console.log(doc.id, doc.data()));
// Batch write
const batch = writeBatch(db);
batch.set(doc(db, 'users', 'bob'), { name: 'Bob', role: 'user' });
batch.set(doc(db, 'users', 'carol'), { name: 'Carol', role: 'user' });
await batch.commit();
// Transaction
const { runTransaction } = await import('firebase/firestore');
await runTransaction(db, async (transaction) => {
const counterDoc = doc(db, 'counters', 'page_views');
const snap = await transaction.get(counterDoc);
transaction.update(counterDoc, { count: (snap.data().count || 0) + 1 });
});
// Firestore Security Rules (firestore.rules)
// rules_version = '2';
// service cloud.firestore {
// match /databases/{database}/documents {
// match /users/{userId} {
// allow read, write: if request.auth != null;
// }
// }
// }| Database | Type | Best For | Scale |
|---|---|---|---|
| Cloud SQL | Managed RDBMS | Existing apps, PostgreSQL/MySQL | Vertical scale |
| Cloud Spanner | Global RDBMS | Global scale, ACID, 99.999% SLA | Horizontal (nodes) |
| Firestore | NoSQL document | Mobile/web apps, real-time sync | Auto-scale |
| Cloud Bigtable | Wide-column NoSQL | IoT, time-series, ML data | PB-scale |
| Memorystore | In-memory (Redis/Memcached) | Caching, session store | GB-TB scale |
| AlloyDB | Postgres-compatible OLAP | Analytics + transactions | Auto-scale |
# ── VPC Network Creation ──
# Create custom VPC
gcloud compute networks create my-app-vpc \
--subnet-mode custom \
--bgp-routing-mode global
# Create subnets (one per region/AZ)
gcloud compute networks subnets create web-subnet \
--network my-app-vpc \
--region us-central1 \
--range 10.1.0.0/24 \
--enable-private-ip-google-access
gcloud compute networks subnets create app-subnet \
--network my-app-vpc \
--region us-central1 \
--range 10.2.0.0/24 \
--enable-private-ip-google-access
gcloud compute networks subnets create db-subnet \
--network my-app-vpc \
--region us-central1 \
--range 10.3.0.0/24 \
--enable-private-ip-google-access \
--enable-flow-logs
# Cloud NAT (for private instances to reach internet)
gcloud compute routers create my-router \
--network my-app-vpc \
--region us-central1
gcloud compute routers nats create my-nat \
--router my-router \
--region us-central1 \
--nat-custom-ips=nat-ip-1,nat-ip-2 \
--nat-all-subnet-ip-ranges
# Firewall rules
gcloud compute firewall-rules create allow-internal \
--network my-app-vpc \
--allow tcp,udp,icmp \
--source-ranges 10.0.0.0/8
gcloud compute firewall-rules create allow-https \
--network my-app-vpc \
--allow tcp:443 \
--source-ranges 0.0.0.0/0 \
--target-tags web-server
gcloud compute firewall-rules create allow-iap-ssh \
--network my-app-vpc \
--allow tcp:22 \
--source-ranges 35.235.240.0/20# ── Global Load Balancer (HTTP/S) ──
# Reserve a global static IP
gcloud compute addresses create my-app-ip --global
# Create health check
gcloud compute health-checks create http my-app-hc \
--port 80 \
--request-path /health
# Create backend service
gcloud compute backend-services create my-app-backend \
--global \
--protocol HTTP \
--health-checks my-app-hc \
--port-name http
# Add NEG (Network Endpoint Group) as backend
gcloud compute network-endpoint-groups create my-app-neg \
--region us-central1 \
--network-endpoint-type serverless \
--cloud-run-service my-service
gcloud compute backend-services add-backend my-app-backend \
--global \
--network-endpoint-group my-app-neg \
--network-endpoint-group-region us-central1
# URL map (routing rules)
gcloud compute url-maps create my-app-lb \
--default-service my-app-backend
# SSL certificate (managed)
gcloud compute ssl-certificates create my-app-cert \
--domains app.example.com \
--global
# HTTPS proxy
gcloud compute target-https-proxies create my-app-https \
--url-map my-app-lb \
--ssl-certificates my-app-cert
# Forwarding rule (binds IP → proxy)
gcloud compute forwarding-rules create my-app-https-rule \
--global \
--address my-app-ip \
--target-https-proxy my-app-https \
--ports 443
# ── Cloud CDN (attach to backend) ──
gcloud compute backend-services update my-app-backend \
--global \
--enable-cdn \
--cache-mode CACHE_ALL_STATIC| Type | Scope | Protocol | Use Case |
|---|---|---|---|
| HTTP(S) LB | Global | HTTP/HTTPS/gRPC | Web apps, global anycast |
| SSL Proxy LB | Global | TCP/TLS | Custom TLS, non-HTTP |
| TCP Proxy LB | Global | TCP | Non-HTTP TCP, gaming, IoT |
| Network LB | Regional | TCP/UDP | High throughput, gaming, DNS |
| Internal LB | Regional | TCP/UDP/HTTP(S) | Internal microservices |
| Concept | Description |
|---|---|
| VPC | Virtual Private Cloud — isolated network |
| Subnet | IP range in a region (supports secondary ranges) |
| Firewall Rules | Stateful L3/L4 (allow/deny by tags/ranges) |
| VPC Flow Logs | Log all network flows for analysis |
| Cloud NAT | Private instances → internet (no public IP) |
| Cloud Armor | WAF + DDoS protection on LB |
| Private Service Connect | Private access to Google APIs |
| VPC Peering | Connect two VPCs (transitive not supported) |
# ── IAM: Identity & Access Management ──
# Create a service account
gcloud iam service-accounts create my-app-sa \
--display-name "My Application SA"
# Grant roles to service account
gcloud projects add-iam-policy-binding my-project \
--member serviceAccount:my-app-sa@my-project.iam.gserviceaccount.com \
--role roles/storage.objectViewer
gcloud projects add-iam-policy-binding my-project \
--member serviceAccount:my-app-sa@my-project.iam.gserviceaccount.com \
--role roles/cloudsql.client
# Grant multiple roles at once
for ROLE in roles/logging.logWriter roles/monitoring.metricWriter roles/trace.agent; do
gcloud projects add-iam-policy-binding my-project \
--member serviceAccount:my-app-sa@my-project.iam.gserviceaccount.com \
--role $ROLE
done
# Key a service account to a GCE / Cloud Run
gcloud compute instances set-service-account my-vm \
--service-account my-app-sa@my-project.iam.gserviceaccount.com \
--zone us-central1-a
# Create & download service account key (avoid if possible)
gcloud iam service-accounts keys create key.json \
--iam-account my-app-sa@my-project.iam.gserviceaccount.com
# ── Workload Identity (GKE → GCP Services, no keys!) ──
# Create IAM binding
gcloud iam service-accounts add-iam-policy-binding \
my-app-sa@my-project.iam.gserviceaccount.com \
--role roles/iam.workloadIdentityUser \
--member "serviceAccount:my-project.svc.id.goog[namespace/kubernetes-sa]"
# In Kubernetes: annotate the service account
kubectl annotate serviceaccount my-k8s-sa \
iam.gke.io/gcp-service-account=my-app-sa@my-project.iam.gserviceaccount.com
# ── List IAM policies
gcloud projects get-iam-policy my-project \
--flatten=bindings \
--filter="bindings.members:my-app-sa" \
--format="table(bindings.role)"| Role Type | Prefix | Description |
|---|---|---|
| Basic | roles/viewer, roles/editor, roles/owner | Coarse-grained, broad access |
| Predefined | roles/storage.objectViewer | Fine-grained, service-specific |
| Custom | projects/my-project/roles/customRole | Tailored permissions |
| Role | Access |
|---|---|
| roles/viewer | Read-only access to all resources |
| roles/editor | Read/write access (no billing/IAM) |
| roles/owner | Full access + billing + IAM |
| roles/storage.admin | Full GCS management |
| roles/cloudsql.client | Connect to Cloud SQL instances |
| roles/logging.logWriter | Write logs to Cloud Logging |
| roles/monitoring.metricWriter | Write metrics to Cloud Monitoring |
| roles/iam.serviceAccountUser | Attach SA to resources |
// ── Cloud Functions (2nd Gen — built on Cloud Run) ──
// index.js
import { HttpFunctionRequest, HttpFunctionResponse } from '@google-cloud/functions-framework';
// HTTP-triggered function
export const helloWorld = (req: HttpFunctionRequest, res: HttpFunctionResponse) => {
const name = req.query.name || req.body?.name || 'World';
res.json({ message: `Hello, ${name}!`, timestamp: new Date().toISOString() });
};
// Event-triggered function (Cloud Storage)
export const processUpload = async (file: any) => {
const { name, bucket, contentType } = file;
console.log(`Processing file: ${name} from ${bucket}`);
// Transform and save to another bucket
// ...
return { status: 'processed', file: name };
};
// Pub/Sub triggered function
export const handleMessage = (message: any, context: any) => {
const data = JSON.parse(Buffer.from(message.data, 'base64').toString());
console.log(`Received message: ${JSON.stringify(data)}`);
console.log(`Message ID: ${context.eventId}`);
console.log(`Publish time: ${context.timestamp}`);
};
// Scheduled function (Cloud Scheduler + Pub/Sub)
export const dailyCleanup = async (event: any) => {
const now = new Date();
console.log(`Running daily cleanup at ${now.toISOString()}`);
// Delete old temp files, aggregate logs, etc.
};# ── Deploy Cloud Functions (2nd Gen) ──
# HTTP function
gcloud functions deploy hello-world \
--gen2 \
--runtime nodejs20 \
--region us-central1 \
--source . \
--entry-point helloWorld \
--trigger-http \
--allow-unauthenticated \
--memory 512Mi \
--timeout 60s \
--min-instances 0 \
--max-instances 1000 \
--set-env-vars NODE_ENV=production
# Event-driven (Cloud Storage)
gcloud functions deploy process-upload \
--gen2 \
--runtime nodejs20 \
--region us-central1 \
--source . \
--entry-point processUpload \
--trigger-event google.storage.object.finalize \
--trigger-resource my-bucket/uploads
# Pub/Sub trigger
gcloud functions deploy handle-message \
--gen2 \
--runtime nodejs20 \
--region us-central1 \
--source . \
--entry-point handleMessage \
--trigger-event google.cloud.pubsub.topic.publish \
--trigger-resource my-topic
# ── App Engine ──
# Deploy a standard environment app
gcloud app deploy --project my-project
# View logs
gcloud functions logs read hello-world --gen2 --limit 50
gcloud app logs tail -s default| Feature | Cloud Functions 2nd Gen | Cloud Run | App Engine |
|---|---|---|---|
| Runtime | Any container (via buildpack) | Any container | Python/Java/Go/Node/PHP |
| Trigger | HTTP, event, schedule | HTTP, job | HTTP, cron, task queue |
| Concurrency | 1 (per instance) | Up to 1000 | Auto |
| Cold start | Medium | Low | Low |
| Min instances | Yes | Yes | Yes |
| Scaling | 0 → N | 0 → N | Auto |
| Max timeout | 60 min | 60 min | 24 hours (tasks) |
| Pricing | Per invocation + compute | Per request + compute | Per instance hour |
| Source | Event Type | Use Case |
|---|---|---|
| HTTP | Request | API endpoints, webhooks |
| Cloud Storage | Object change | Image resize, data processing |
| Pub/Sub | Message published | Async processing, fan-out |
| Cloud Scheduler | Cron schedule | Periodic cleanup, reports |
| Firestore | Document write | Trigger on data change |
| Firebase Auth | User event | Welcome emails, audit logs |
# ── Pub/Sub (Asynchronous Messaging) ──
# Create a topic
gcloud pubsub topics create user-events
# Create subscription (pull)
gcloud pubsub subscriptions create user-events-processor \
--topic user-events \
--ack-deadline 60 \
--max-retry-delays 10s,30s,60s
# Publish a message
gcloud pubsub topics publish user-events --message='{
"userId": "u123",
"event": "page_view",
"timestamp": "2025-01-15T10:30:00Z"
}'
# Publish from code (Python)
# from google.cloud import pubsub_v1
# publisher = pubsub_v1.PublisherClient()
# topic_path = publisher.topic_path('my-project', 'user-events')
# publisher.publish(topic_path, b'{"event": "click"}', type='user_action')
# Pull messages
gcloud pubsub subscriptions pull user-events-processor \
--limit 10 --auto-ack
# Dead letter topic (for failed messages)
gcloud pubsub topics create user-events-dlq
gcloud pubsub subscriptions update user-events-processor \
--dead-letter-policy-topic=user-events-dlq \
--dead-letter-policy-max-delivery-attempts 5# ── Dataflow (Apache Beam Pipeline) ──
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
options = PipelineOptions(
runner='DataflowRunner',
project='my-project',
region='us-central1',
temp_location='gs://my-bucket/temp',
staging_location='gs://my-bucket/staging',
)
with beam.Pipeline(options=options) as p:
(
p
# Read from Pub/Sub
| 'Read PubSub' >> beam.io.ReadFromPubSub(topic='projects/my-project/topics/user-events')
# Parse JSON
| 'Parse JSON' >> beam.Map(lambda x: json.loads(x.decode('utf-8')))
# Window: fixed 1-minute windows
| 'Window' >> beam.WindowInto(beam.window.FixedWindows(60))
# Group by event type and count
| 'Count by Type' >> beam.CombinePerKey(beam.counts.PerKey())
# Format output
| 'Format' >> beam.Map(lambda kv: {'event_type': kv[0], 'count': kv[1]})
# Write to BigQuery
| 'Write BigQuery' >> beam.io.WriteToBigQuery(
table='my-project:analytics.event_counts',
schema='event_type:STRING,count:INTEGER',
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
)
)| Service | Type | Best For |
|---|---|---|
| Pub/Sub | Message broker | Event streaming, async processing |
| Dataflow | Batch + streaming ETL | Complex data transformations |
| BigQuery | Data warehouse | SQL analytics, BI dashboards |
| Dataproc | Managed Spark/Hadoop | Big data processing, ML |
| Data Fusion | Visual ETL | No-code data integration |
| Composer | Managed Airflow | Workflow orchestration, DAGs |
| Feature | Pub/Sub | Cloud Tasks |
|---|---|---|
| Pattern | Pub/sub (fan-out) | Task queue (work distribution) |
| Delivery | Push or pull | Push (HTTP target) |
| Ordering | Best effort | Strict ordering per queue |
| Retries | Configurable | Exponential backoff built-in |
| Best for | Event-driven, multi-subscriber | Async HTTP jobs, rate-limited APIs |
# ── Billing & Cost Management ──
# Link billing account to project
gcloud beta billing projects link my-project \
--billing-account 01ABCD-234EFG-567HIJ
# Set budget alert
gcloud billing budgets create \
--billing-account 01ABCD-234EFG-567HIJ \
--display-name "Monthly Budget Alert" \
--budget-amount 500USD \
--threshold-rule=percent=50 \
--threshold-rule=percent=90 \
--threshold-rule=percent=100
# View current costs (via bq)
bq query --use_legacy_sql=false '''
SELECT
service.description as service,
SUM(cost) as total_cost,
SUM((SELECT SUM(amount) FROM UNNEST(credits))) as credits
FROM `my-project.billing.gcp_billing_export_v1_XXXXXX`
WHERE billing_account_id = "01ABCD-234EFG-567HIJ"
AND usage_start_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY)
GROUP BY service
ORDER BY total_cost DESC
LIMIT 20
'''
# Enable Recommender (cost optimization)
gcloud recommender recommendations list \
--project my-project \
--recommender google.compute.machineTypeRecommender \
--format='yaml(content.operationGroups[0].operations[0].path, content.operationGroups[0].operations[0].value, costSavings.cost)'
# ── Cost Optimization Checklist ──
# 1. Use committed use discounts (1yr or 3yr) for predictable workloads
# 2. Use preemptible/spot VMs for fault-tolerant jobs (up to 90% cheaper)
# 3. Right-size GCE instances with Recommender
# 4. Use Autopilot GKE (pay per pod, no idle nodes)
# 5. Set min-instances=0 for Cloud Run (scale to zero)
# 6. Use lifecycle policies on GCS (auto-tier to cheaper storage)
# 7. Delete unused static IPs ($7/mo for reserved unused IPs)
# 8. Enable Always Free tier resources where possible| Model | Description | Example |
|---|---|---|
| On-demand | Pay per use, no commitment | GCE, Cloud Run, Functions |
| Committed Use | 1 or 3-year contract, up to 57% off | GCE, Cloud SQL, BigQuery |
| Spot / Preemptible | Up to 91% off, can be reclaimed | GCE VMs for batch jobs |
| Sustained Use | Automatic discount for steady usage | GCE (applied automatically) |
| Free Tier | Always Free resources per month | e2-micro, 5GB GCS, BigQuery queries |