ql.py

import requests
import json
import time

URL = "https://graph.cars.com/graphql/api"

HEADERS = {
    "content-type": "application/json",
    "x-api-key": "5rrmnWVl1MDzPcEnDvEp3Pu101IGXEGo",
    "user-agent": "Mozilla/5.0"
}

GRAPHQL_QUERY =  """query SearchResultsPageSearch($agenticSearchQuery: String, $aiSearchId: String, $page: Int, $pageSize: Int, $searchInstanceId: String, $selectedSearchFilters: [SelectedSearchFilterInput!]!, $sort: ListingSearchSortField) {
  srpSearch(
    additionalSupportedResultComponents: [BRANDED_CANVAS_AD, INFO_PANEL]
    agenticSearchQuery: $agenticSearchQuery
    aiSearchId: $aiSearchId
    page: $page
    pageSize: $pageSize
    searchInstanceId: $searchInstanceId
    selectedSearchFilters: $selectedSearchFilters
    sort: $sort
  ) {
    ...SearchResultsProperties
  }
}

fragment AnalyticsContextProperties on AnalyticsContext {
  context
  fingerprint
}

fragment StackProperties on Stack {
  __typename
  alignItems
  contentReference {
    contentId
    groupId
  }
  customGap
  direction
  gap
  justifyContent
  wrap
}

fragment NavigateButtonProperties on NavigateButton {
  actionLabel
  actionTarget
  prefixIcon
  suffixIcon
  analyticsContext {
    ...AnalyticsContextProperties
  }
}

fragment AnalyticsInteractionProperties on AnalyticsInteraction {
  interactionType
  payload
  trid
}

fragment NavigateInteractionProperties on NavigateInteraction {
  clientAttrs {
    name
    value
  }
  destination
  interactionType
  target
}

fragment DniInteractionProperties on DniInteraction {
  dniCorrelationData
  fallbackNumber
  interactionType
  listingId
}

fragment OpenLeadFormInteractionProperties on OpenLeadFormInteraction {
  interactionType
  action
}

fragment OpenMakeOfferFormInteractionProperties on OpenMakeOfferFormInteraction {
  interactionType
  action
  target
  url
}

fragment KeyValuePairProperties on KeyValuePair {
  key
  value
}

fragment GraphqlQueryProperties on GraphqlQuery {
  queryParams {
    ...KeyValuePairProperties
  }
  uri
}

fragment OpenPopoverInteractionProperties on OpenPopoverInteraction {
  appearance
  content {
    ... on GraphqlQuery {
      ...GraphqlQueryProperties
    }
  }
  interactionType
  target
}

fragment CarsonSearchChatInteractionProperties on CarsonSearchChatInteraction {
  optimisticMessage {
    id
    content
    createdAt
    isStreaming
    role
  }
  carsonActionPayload
}

fragment ChangeSearchFiltersInteractionProperties on ChangeSearchFiltersInteraction {
  interactionType
  selectedSearchFiltersAdd {
    filter
    enabled
    point {
      srid
      coordinates {
        latitude
        longitude
      }
    }
    radiusMiles
    value
    values
    zipCode
  }
  selectedSearchFiltersRemove {
    filter
    enabled
    point {
      srid
      coordinates {
        latitude
        longitude
      }
    }
    radiusMiles
    value
    values
    zipCode
  }
}

fragment HasClickInteractionsProperties on HasClickInteractions {
  onClickInteractions {
    ...AnalyticsInteractionProperties
    ...NavigateInteractionProperties
    ...DniInteractionProperties
    ...OpenLeadFormInteractionProperties
    ...OpenMakeOfferFormInteractionProperties
    ...OpenPopoverInteractionProperties
    ...CarsonSearchChatInteractionProperties
    ...ChangeSearchFiltersInteractionProperties
  }
}

fragment TextSnippetProperties on TextSnippet {
  ...HasClickInteractionsProperties
  fontColor
  text
  textStyle
}

fragment TextProperties on Text {
  headingLevel
  textSnippets {
    ...TextSnippetProperties
  }
}

fragment BadgeProperties on Badge {
  analyticsContext {
    ...AnalyticsContextProperties
  }
  icon
  variant
  strong
  explainer {
    button {
      ...NavigateButtonProperties
    }
    body {
      ...TextProperties
    }
    title
  }
  description
  value
}

fragment HasViewInteractionsProperties on HasViewInteractions {
  onViewInteractions {
    ...AnalyticsInteractionProperties
  }
}

fragment ButtonProperties on Button {
  __typename
  accessibilityLabel
  analyticsContext {
    ...AnalyticsContextProperties
  }
  ...HasClickInteractionsProperties
  ...HasViewInteractionsProperties
  displayLabel
  enabled
  label
  prefixIcon
  suffixIcon
  size
  variant
}

fragment DatumIconProperties on DatumIcon {
  icon
  name
  styleIdentifier
  value
}

fragment InlineLinkProperties on InlineLink {
  actionTarget
  analyticsContext {
    ...AnalyticsContextProperties
  }
  destinationUrl
  length
  location
}

fragment DisplayValueProperties on DisplayValue {
  links {
    ...InlineLinkProperties
  }
  text
}

fragment DatumDisplayProperties on DatumDisplay {
  label
  iconName
  tooltip {
    title
    value
  }
  text {
    ...TextProperties
  }
  displayValue {
    text
    links {
      analyticsContext {
        ...AnalyticsContextProperties
      }
      destinationUrl
      length
      location
    }
  }
}

fragment MonthlyPaymentDetailsProperties on MonthlyPaymentDetails {
  disclaimer {
    ...DisplayValueProperties
  }
  datums {
    ...DatumDisplayProperties
  }
  title
}

fragment MonthlyPaymentProperties on MonthlyPayment {
  analyticsContext {
    ...AnalyticsContextProperties
  }
  label
  icon
  popover {
    ...MonthlyPaymentDetailsProperties
  }
}

fragment CardStackChildren on Element {
  __typename
  ... on Badge {
    ...BadgeProperties
  }
  ... on Button {
    ...ButtonProperties
  }
  ... on DatumIcon {
    ...DatumIconProperties
  }
  ... on MonthlyPayment {
    ...MonthlyPaymentProperties
  }
  ... on Text {
    ...TextProperties
  }
}

fragment StackLevel4 on Stack {
  ...StackProperties
  items {
    ...CardStackChildren
  }
}

fragment StackLevel3 on Stack {
  ...StackProperties
  items {
    ...CardStackChildren
    ... on Stack {
      ...StackLevel4
    }
  }
}

fragment StackLevel2 on Stack {
  ...StackProperties
  items {
    ...CardStackChildren
    ... on Stack {
      ...StackLevel3
    }
  }
}

fragment StackWithChildren on Stack {
  ...StackProperties
  items {
    ...CardStackChildren
    ... on Stack {
      ...StackLevel2
    }
  }
}

fragment DescriptionListProperties on DescriptionList {
  __typename
  items {
    label
    value
  }
}

fragment UnionBoxStringProperties on UnionBoxString {
  string
}

fragment RatingReviewsProperties on RatingReviews {
  __typename
  ...NavigateButtonProperties
  ...UnionBoxStringProperties
}

fragment RatingProperties on Rating {
  action {
    ...RatingReviewsProperties
  }
  compact
  rating
  size
  variant
}

fragment BreakdownItemProperties on BreakdownItem {
  breakdownText {
    ...TextProperties
  }
  breakdownRating {
    ...RatingProperties
  }
}

fragment ReviewProperties on Review {
  __typename
  breakdown {
    ...BreakdownItemProperties
  }
  closeReviewRevealButton
  date
  openReviewRevealButton
  reviewBody
  reviewBulletPoints {
    ...TextProperties
  }
  subjectName
  title
  userAttrs
  userInfo
  rating {
    ...RatingProperties
  }
  vehicleRating {
    ...RatingProperties
  }
}

fragment SearchResultsProperties on SrpResults {
  analytics {
    ...AnalyticsContextProperties
  }
  adTargetingParameters {
    key
    value
  }
  metadata {
    pageSize
    page
    selectedSearchFilters {
      enabled
      filter
      radiusMiles
      value
      values
      zipCode
    }
    seoAggregations
    sort
    totalListings
    totalPages
  }
  searchTitle
  results {
    ... on SrpListingGridCard {
      __typename
      analytics {
        ...AnalyticsContextProperties
      }
      body {
        ... on Stack {
          ...StackWithChildren
        }
      }
      footer {
        ... on Stack {
          ...StackWithChildren
        }
      }
      gallery {
        images {
          url
          altText
        }
        imageCount
      }
      attributionType
      extendedVehicleInfo {
        features {
          ...DescriptionListProperties
        }
        dealerReview {
          ...ReviewProperties
        }
        vehicleReview {
          ...ReviewProperties
        }
        vehicleInformation {
          ...DescriptionListProperties
        }
      }
      isSaved
      listingId
      newVdpEligible
      onViewInteractions {
        ...AnalyticsInteractionProperties
      }
      sponsoredType
    }
    ... on AdUnitModule {
      __typename
      adUnitPath
      adType
      adSubtype
      adSizeLayout {
        adSizes {
          height
          width
        }
        fluid
      }
      adAppnexusPlacementId
      adId
      adHiddenOn
    }
    ... on BrandedCanvasAd {
      __typename
      body
      creativeId
      requestUrl
      make
      model
    }
    ... on InfoPanel {
      __typename
      body {
        __typename
        ...TextProperties
      }
      disclaimer {
        trigger
        body {
          __typename
          ...TextProperties
        }
      }
      icon
      title
    }
    ... on Text {
      __typename
      ...TextProperties
    }
  }
  savedSearchIdentifier
  searchInstanceId
  sortSelector {
    buttonTitle
    options {
      isSelected
      name
      value
    }
  }
}"""

BASE_VARIABLES = {
    "pageSize": 22,
    "searchInstanceId": "78bbfab6-cf33-433b-9304-da2c36891964",
    "selectedSearchFilters": [
        {
            "enabled": None,
            "value": None,
            "values": ["nissan-pathfinder"],
            "filter": "models",
            "zipCode": None,
            "radiusMiles": None
        },
        {
            "enabled": None,
            "value": None,
            "values": None,
            "filter": "area",
            "zipCode": "60606",
            "radiusMiles": 50
        },
        {
            "enabled": None,
            "value": None,
            "values": ["nissan"],
            "filter": "makes",
            "zipCode": None,
            "radiusMiles": None
        },
        {
            "enabled": None,
            "value": "new",
            "values": None,
            "filter": "stock_type",
            "zipCode": None,
            "radiusMiles": None
        }
    ],
    "sort": "BEST_MATCH_DESC"
}


def fetch_page(page: int):
    payload = {
        "operationName": "SearchResultsPageSearch",
        "query": GRAPHQL_QUERY,
        "variables": {
            **BASE_VARIABLES,
            "page": page
        }
    }

    res = requests.post(URL, headers=HEADERS, json=payload, timeout=30)
    
    res.raise_for_status()
    return res.json()


def is_end_of_results(results):
    END_TITLES = {
        "No exact matches found",
        "You’ve reached the end of results",
    }

    for item in results:
        if item.get("__typename") != "InfoPanel":
            continue

        # 1️⃣ Title-based stop (most reliable)
        title = item.get("title")
        if title in END_TITLES:
            return True

        # 2️⃣ Body text-based stop (fallback)
        body = item.get("body", {})
        if body.get("__typename") == "Text":
            snippets = body.get("textSnippets", [])
            for snippet in snippets:
                text = snippet.get("text", "")
                if "saving your search" in text.lower():
                    return True

    return False


def extract_listings(results):
    listings = []
    for item in results:
        if item.get("__typename") == "SrpListingGridCard":
            listings.append({
                "listingId": item.get("listingId"),
                "gallery": item.get("gallery"),
                "body": item.get("body"),
                "footer": item.get("footer"),
                "extendedVehicleInfo": item.get("extendedVehicleInfo")
            })
    return listings


def crawl_all_pages():
    page = 1
    all_listings = []

    while True:
        print(f"Fetching page {page}")
        data = fetch_page(page)

        results = data["data"]["srpSearch"]["results"]

        if is_end_of_results(results):
            print("Reached end of results.")
            break

        page_listings = extract_listings(results)
        print(f"Found {len(page_listings)} listings")

        all_listings.extend(page_listings)
        page += 1

        time.sleep(1)  # polite delay

    return all_listings



if __name__ == "__main__":
    listings = crawl_all_pages()

    with open("cars_results.json", "w", encoding="utf-8") as f:
        json.dump(listings, f, indent=2)

    print(f"Saved {len(listings)} listings to cars_results.json")