Different HTML Responses Between httpx and requests for Same Request

[lvlh2]

Description

I encountered an issue where using identical headers and accessing the same URL, the responses from httpx and requests are different. Specifically, when scraping images search results from Bing, the data-nexturl attribute extracted from the HTML response does not update correctly with httpx, while it works as expected with requests.

Code to Reproduce

Using httpx (not working correctly)

import json
import time
from urllib.parse import urljoin

import httpx
from lxml.html import fromstring

KEYWORD = "race car"
NUM_IMAGES = 400

def get_urls(client: httpx.Client, next_url: str | None = None) -> dict:
    if next_url is None:
        url = "https://cn.bing.com/images/search"
        params = {
            "q": KEYWORD,
            "go": "Search",
            "qs": "ds",
            "form": "QBIR",
            "first": "1",
        }
        response = client.get(url, params=params)
        response.raise_for_status()
    else:
        response = client.get(urljoin("https://cn.bing.com", next_url))
        response.raise_for_status()

    doc = fromstring(response.text)

    next_url = doc.xpath('//div[@class="dgControl dtl hover"]/@data-nexturl')[0].strip()
    assert next_url, "No more images"

    print(next_url)

    image_urls = [
        json.loads(item)["murl"] for item in doc.xpath('//div[@class="imgpt"]/a/@m')
    ]

    return {"next_url": next_url, "image_urls": image_urls}

def main():
    headers = {
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0",
    }
    with httpx.Client(headers=headers) as client:
        urls = get_urls(client)
        image_urls: list[str] = urls["image_urls"]

        client.headers["accept"] = "*/*"
        while len(image_urls) < NUM_IMAGES:
            urls = get_urls(client, next_url=urls["next_url"])
            image_urls.extend(urls["image_urls"])

            time.sleep(5)

if __name__ == "__main__":
    main()

Output:

/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&datsrc=I&layout=RowBased_Landscape&apc=0&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*32_r*9
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*63_r*18
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*94_r*27
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*125_r*36
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*156_r*45
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*187_r*54
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*218_r*63
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*249_r*72
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*280_r*81
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*311_r*90
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*342_r*99
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*373_r*108
/images/async?q=race+car&first=32&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAqwxVseNfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkSFBERCBiBiiYw95dhFBiEGAo5QHUAqRiMeQBwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*404_r*117

Note that the first parameter remains at 32 and does not update correctly.

Using requests (working correctly)

import json
import time
from urllib.parse import urljoin

import requests
from lxml.html import fromstring

KEYWORD = "race car"
NUM_IMAGES = 400


def get_urls(session: requests.Session, next_url: str | None = None) -> dict:
    if next_url is None:
        url = "https://cn.bing.com/images/search"
        params = {
            "q": KEYWORD,
            "go": "Search",
            "qs": "ds",
            "form": "QBIR",
            "first": "1",
        }
        response = session.get(url, params=params)
        response.raise_for_status()
    else:
        response = session.get(urljoin("https://cn.bing.com", next_url))
        response.raise_for_status()

    doc = fromstring(response.text)

    next_url = doc.xpath('//div[@class="dgControl dtl hover"]/@data-nexturl')[0].strip()
    assert next_url, "No more images"

    print(next_url)

    image_urls = [
        json.loads(item)["murl"] for item in doc.xpath('//div[@class="imgpt"]/a/@m')
    ]

    return {"next_url": next_url, "image_urls": image_urls}


def main():
    headers = {
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0",
    }
    with requests.Session() as session:
        session.headers.update(headers)
        urls = get_urls(session)
        image_urls: list[str] = urls["image_urls"]

        session.headers["accept"] = "*/*"
        while len(image_urls) < NUM_IMAGES:
            urls = get_urls(session, next_url=urls["next_url"])
            image_urls.extend(urls["image_urls"])

            time.sleep(5)


if __name__ == "__main__":
    main()

Output:

/images/async?q=race+car&first=31&count=35&cw=1024&ch=768&relp=30&datsrc=I&layout=RowBased_Landscape&apc=0&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAaUM0UONfTNkSEFAIUgAACx%2bvDCtYBLCmBaDbAoZkQFBERCBiBiiYw95ZhFBiEGAo5QHUAqRiMeQAwAAAAAAAAA%3d%3d&mmasync=1&dgState=x*626_y*1531_h*183_c*2_i*31_r*9
/images/async?q=race+car&first=63&count=35&cw=1024&ch=768&relp=32&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAA9vn%2fduNfTNne6EVIAgEYhOHKGYOCKs6IAkhhg4gQSBCVl%2biCjQFgUkI0iCPsxTwIAihAAMADDEBd%2bwAAAAAAAA%3d%3d&mmasync=1&dgState=x*313_y*1509_h*183_c*1_i*63_r*18
/images/async?q=race+car&first=93&count=35&cw=1024&ch=768&relp=30&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAFGppvONfTNnwpQRItdlKpYEUBWwoh5QIAVFCEEF0hwEiYUImIYN5IChYBkIkQwwiIMICRASAJBQaywAAAAAAAA%3d%3d&mmasync=1&dgState=x*591_y*1351_h*185_c*3_i*93_r*26
/images/async?q=race+car&first=111&count=35&cw=1024&ch=768&relp=18&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAJYzbR%2bNfTNmCwBAEAIERBEARAAIAISEYQ4AYgQQAAKAgCJQAwynwFgAkgECrCRABTEDAGIAgCBBIkAAAAAAAAA%3d%3d&mmasync=1&dgState=x*284_y*752_h*184_c*1_i*111_r*31
/images/async?q=race+car&first=134&count=35&cw=1024&ch=768&relp=23&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAU4W%2f3ONfTNl2pQRItFFIrIAUASgghDwBAQMgEEAUhAAiYUICIIFiACBIDAOgQxgiOYICQACADB0K0AAAAAAAAA%3d%3d&mmasync=1&dgState=x*414_y*942_h*184_c*2_i*134_r*37
/images/async?q=race+car&first=159&count=35&cw=1024&ch=768&relp=25&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAArNbABONfTNmCgBFEA4ARAAATAgoGKaNKS4MYjBEMqCKoGhRAUC2QBggEAHFrSVBLTECAHKAgCBLgkAAAAAAAAA%3d%3d&mmasync=1&dgState=x*282_y*1129_h*183_c*1_i*159_r*44
/images/async?q=race+car&first=184&count=35&cw=1024&ch=768&relp=25&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAA2q%2fkv%2bNfTNkW5LMAgGMECABlgAkgBDghAEIiAIKTDgQAIYKhTggGDhAAiKGJAVMrWWQZA0SICAshUQAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*4_i*184_r*50
/images/async?q=race+car&first=211&count=35&cw=1024&ch=768&relp=27&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAfkZO%2buNfTNkQgA1UI8EAMAASEAgOTINGWIsZFRUcogK7EyFYEBQQNggAILPNQFpKFAyEBCBiKwLkAQAAAAAAAA%3d%3d&mmasync=1&dgState=x*333_y*1322_h*182_c*1_i*211_r*58
/images/async?q=race+car&first=239&count=35&cw=1024&ch=768&relp=28&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAjO%2bx%2feNfTNkRwBMCjOEkACJlksEQBAA0AWAC0IaJDHAAOAKBTytjDhGDhLiJ0dIuQESQA4UADUIhRQAAAAAAAA%3d%3d&mmasync=1&dgState=x*271_y*1128_h*183_c*1_i*239_r*65
/images/async?q=race+car&first=264&count=35&cw=1024&ch=768&relp=25&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAwt6qauNfTNmEUCaQI0AAEGERGwCcaAGEEQ4TEQSAosliEKEZABBQMggAKAIAMAyArBxaZBDgJwoFEAAAAAAAAA%3d%3d&mmasync=1&dgState=x*354_y*1156_h*186_c*1_i*264_r*72
/images/async?q=race+car&first=286&count=35&cw=1024&ch=768&relp=22&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAArdItZ%2bNfTNkFAAoKeIQVgDJESICAggAnAWQAEhAMAAcAiALJAQOGHClAJCgJ0oBwwAAIgkARggAAQQAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*4_i*286_r*77
/images/async?q=race+car&first=314&count=35&cw=1024&ch=768&relp=28&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAfIF5CeNfTNm01jawA0jQJGcRE6CQKiiFEYYRMEDAAstDUAgZABBUiipoACIydAQAqKhQZxvABA4FEAAAAAAAAA%3d%3d&mmasync=1&dgState=x*578_y*1321_h*183_c*2_i*314_r*85
/images/async?q=race+car&first=345&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAEyHVGeNfTNmGAWJKeYQzhJFFCAConhBjQAQAGqEEIBcN4ILZcErEHExgOCwIZtFR6BkKgNgQ4oA2wAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*345_r*93
/images/async?q=race+car&first=376&count=35&cw=1024&ch=768&relp=31&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAMviT0uNfTNmQhlkQEkAwbGVIFKAgoK%2bBxAZflCFJAAJLBAAYTFdQCCokR4Y6VBJRURuEQQrwZESIqAAAAAAAAA%3d%3d&mmasync=1&dgState=x*0_y*0_h*0_c*3_i*376_r*101
/images/async?q=race+car&first=405&count=35&cw=1024&ch=768&relp=29&apc=0&datsrc=I&layout=RowBased_Landscape&imgbf=DfCtqwgAAACQAQAAAAAAAAAAAAAFAAAAHzSE9eNfTNkQFTwQ7giyCREvEiFAlBEAcGAWFI1sQAkFAKVYUjDpCEUAMHUANZKgsBw8AEhUbSIEIAAAAAAAAA%3d%3d&mmasync=1&dgState=x*238_y*1138_h*184_c*1_i*405_r*108

Here, the first parameter updates correctly (31, 63, etc.), allowing the page turning to work.

Environment

• Python Version: 3.13.5
• httpx Version: 0.28.1
• requests Version: 2.32.5
• OS: Windows 11