pass user-agent to api

Author: maslianok

app/api/graphql/route.ts

@@ -1,5 +1,6 @@
 import { type NextRequest, NextResponse } from 'next/server';
 
+import { ORIGINAL_USER_AGENT_HEADER } from '@/app/app.consts';
 import { request } from '@/lib/graphql/request';
 import { rejectNotAllowedGraphqlOperations } from '@/utils/reject-not-allowed-graphql-operations';
 import { rejectWrongOrigin } from '@/utils/reject-wrong-origin';
@@ -16,7 +17,8 @@ export async function POST(req: NextRequest) {
       return NextResponse.json({ error: 'Forbidden' }, { status: 403 });
     }
 
-    const { data, status } = await request(query, variables);
+    const originalUserAgent = req.headers.get(ORIGINAL_USER_AGENT_HEADER) ?? req.headers.get('user-agent') ?? undefined;
+    const { data, status } = await request(query, variables, { originalUserAgent });
 
     return NextResponse.json(data, { status });
   } catch {

app/app.consts.ts

@@ -38,8 +38,6 @@ export const RANK_DESCRIPTIONS = {
   },
 };
 
-export const DEFAULT_LANGUAGE_COLOR = '#64748B';
-
 export const LANGUAGE = {
   description:
     'See which programming languages are the most popular worldwide. Browse detailed summaries by stars and code size - and dive into rankings for each language, globally or by country.',
@@ -51,3 +49,6 @@ export const LANGUAGE = {
       'How many users with at least one public repo with 5+ stars have this language in their public repositories.',
   },
 };
+
+export const DEFAULT_LANGUAGE_COLOR = '#64748B';
+export const ORIGINAL_USER_AGENT_HEADER = 'x-original-user-agent';

lib/graphql/graphql-client.ts

@@ -1,9 +1,10 @@
 import { print } from 'graphql';
 
+import { ORIGINAL_USER_AGENT_HEADER } from '@/app/app.consts';
 import type DocumentNode from '@/types/typed-document-node';
 
 const resolveGraphqlProxyUrl = () => {
-  if (typeof window !== 'undefined') {
+  if (globalThis.window !== undefined) {
     return '/api/graphql';
   }
 
@@ -14,19 +15,40 @@ const resolveGraphqlProxyUrl = () => {
   throw new Error('Cannot resolve GraphQL proxy URL. Set NEXT_PUBLIC_URI.');
 };
 
+const resolveOriginalUserAgent = async () => {
+  if (globalThis.window !== undefined) {
+    return null;
+  }
+
+  try {
+    const { headers } = await import('next/headers');
+    const incomingHeaders = await headers();
+    return incomingHeaders.get('user-agent');
+  } catch {
+    return null;
+  }
+};
+
 export async function graphqlClient<TData, TVariables>(
   document: DocumentNode<TData, TVariables>,
   variables?: TVariables,
   params?: RequestInit,
 ): Promise<TData> {
   const query = print(document);
   const url = resolveGraphqlProxyUrl();
+  const originalUserAgent = await resolveOriginalUserAgent();
+  const requestHeaders = new Headers(params?.headers);
+
+  requestHeaders.set('Content-Type', 'application/json');
+  if (originalUserAgent) {
+    requestHeaders.set(ORIGINAL_USER_AGENT_HEADER, originalUserAgent);
+  }
 
   const res = await fetch(url, {
+    ...params,
     method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
+    headers: requestHeaders,
     body: JSON.stringify({ query, variables }),
-    ...params,
   });
 
   const rawBody = await res.text();

lib/graphql/request.ts

@@ -1,21 +1,33 @@
 'use server';
 
+import { ORIGINAL_USER_AGENT_HEADER } from '@/app/app.consts';
+
 import { signedFetch } from '../signed-fetch';
 
+type RequestOptions = {
+  revalidate?: number;
+  originalUserAgent?: string;
+};
+
 export async function request(
   query: string,
   variables?: Record<string, unknown>,
-  params?: { revalidate?: number },
+  options?: RequestOptions,
 ): Promise<{ data: unknown; status: number }> {
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+    'nextjs-build-phase': String(process.env.NEXT_PHASE === 'phase-production-build'),
+  };
+
+  if (options?.originalUserAgent) {
+    headers[ORIGINAL_USER_AGENT_HEADER] = options.originalUserAgent;
+  }
+
   const response = await signedFetch('/graphql', {
     method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      // Skip rate limit if custom header is present
-      'nextjs-build-phase': String(process.env.NEXT_PHASE === 'phase-production-build'),
-    },
+    headers,
     body: JSON.stringify({ query, variables }),
-    next: { revalidate: params?.revalidate },
+    next: { revalidate: options?.revalidate },
   });
 
   if (!response.ok) {

public/robots.txt

@@ -1,9 +1,14 @@
-User-agent: DataForSeoBot
-Crawl-delay: 60
-Disallow:
+User-agent: Googlebot
+Allow: /
 
+User-agent: Bingbot
+Allow: /
+
+# Generic rule for everyone else
 User-agent: *
-Disallow:
+Crawl-delay: 10
+Disallow: /api/
+Allow: /
 
 Sitemap: https://gitranks.com/profile/_/sitemap.xml
-Sitemap: https://gitranks.com/by/_/sitemap.xml
+Sitemap: https://gitranks.com/by/_/sitemap.xml