In [1]:

import json
import re
import chardet
import requests

In [2]:

def get_text_from_html(url):
    """
    獲取網頁內容，匹配有地址的文本
    :param url: 獲取地址的url
    :return: 返回包含地址的網頁內容
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    content = resp.text

    prefix_ip = 'var ip_result = '
    for row in re.split('\\r\\n\\t', content):
        if prefix_ip in row and row:
            return row

In [3]:

ip = '180.97.125.228'
url = 'https://www.ip138.com/iplookup.asp?ip=%s&action=1' % ip
row = get_text_from_html(url)
row

Out [3]:

'\\t\\tvar ip_result = {"ASN1éê?μ?":"?\\xad??ê?????êD  μ?D? ", "iP??":"180.97.107.0 - 180.97.135.255", "??èYIPv6μ??·":"::B461:7DE4", "ó3é?IPv6μ??·":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"?D1ú", "prov":"?\\xad??ê?", "city":"????êD", "area":"", "idc":"", "yunyin":"μ?D?", "net":""}], "zg":1};'

In [4]:

# 這個網頁文字的編碼是unicode形式的gbk編碼內容
# Unicode codepoints U+0000 to U+00FF和latin-1一一對應，編碼后保留了等價的字節流數據。
row_byte = row.encode("latin-1")
row_byte

Out [4]:

b'\\t\\tvar ip_result = {"ASN\\xb9\\xe9\\xca\\xf4\\xb5\\xd8":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1\\xc4\\xcf\\xbe\\xa9\\xca\\xd0  \\xb5\\xe7\\xd0\\xc5 ", "iP\\xb6\\xce":"180.97.107.0 - 180.97.135.255", "\\xbc\\xe6\\xc8\\xddIPv6\\xb5\\xd8\\xd6\\xb7":"::B461:7DE4", "\\xd3\\xb3\\xc9\\xe4IPv6\\xb5\\xd8\\xd6\\xb7":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"\\xd6\\xd0\\xb9\\xfa", "prov":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1", "city":"\\xc4\\xcf\\xbe\\xa9\\xca\\xd0", "area":"", "idc":"", "yunyin":"\\xb5\\xe7\\xd0\\xc5", "net":""}], "zg":1};'

In [5]:

# 通過字節碼獲取當前的編碼為gb2312
encoding_detect = chardet.detect(row_byte)
encoding_detect

Out [5]:

{'encoding': 'GB2312', 'confidence': 0.99, 'language': 'Chinese'}

In [6]:

# 因此對其進行解碼即可還原最初的unicode字符。
row_target = row_byte.decode("gb2312")
row_target

Out [6]:

'\\t\\tvar ip_result = {"ASN歸屬地":"江蘇省南京市  電信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中國", "prov":"江蘇省", "city":"南京市", "area":"", "idc":"", "yunyin":"電信", "net":""}], "zg":1};'

In [7]:

# 截取文本，轉化為規范的json字符串
json_str = row_target.strip().replace('var ip_result = ', '').replace(';', '')
json_str

Out [7]:

'{"ASN歸屬地":"江蘇省南京市  電信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中國", "prov":"江蘇省", "city":"南京市", "area":"", "idc":"", "yunyin":"電信", "net":""}], "zg":1}'

In [8]:

# 轉化為python里面的數據格式：字典
location_dict = json.loads(json_str)
location_dict

Out [8]:

{'ASN歸屬地': '江蘇省南京市  電信 ',
 'iP段': '180.97.107.0 - 180.97.135.255',
 '兼容IPv6地址': '::B461:7DE4',
 '映射IPv6地址': '::FFFF:B461:7DE4',
 'ip_c_list': [{'begin': 3026283264,
   'end': 3026290687,
   'ct': '中國',
   'prov': '江蘇省',
   'city': '南京市',
   'area': '',
   'idc': '',
   'yunyin': '電信',
   'net': ''}],
 'zg': 1}

In [9]:

# 獲取中文地址
location = location_dict.get('ASN歸屬地', '').strip()
location

Out [9]:

'江蘇省南京市  電信'