#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time : 2018/11/22 
# @Author : liangk
# @Site :
# @File : get_douban_books.py
# @Software: PyCharm

from bs4 import BeautifulSoup
import datetime
import requests
import json
import random

ip_random = -1
article_tag_list = []
article_type_list = []


def get_html(url):
 header = {
  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
 }
 global ip_random
 ip_rand, proxies = get_proxie(ip_random)
 print(proxies)
 try:
  request = requests.get(url=url, headers=header, proxies=proxies, timeout=3)
 except:
  request_status = 500
 else:
  request_status = request.status_code
 print(request_status)
 while request_status != 200:
  ip_random = -1
  ip_rand, proxies = get_proxie(ip_random)
  print(proxies)
  try:
   request = requests.get(url=url, headers=header, proxies=proxies, timeout=3)
  except:
   request_status = 500
  else:
   request_status = request.status_code
  print(request_status)
 ip_random = ip_rand
 request.encoding = 'gbk'
 html = request.content
 print(html)
 return html


def get_proxie(random_number):
 with open('ip.txt', 'r') as file:
  ip_list = json.load(file)
  if random_number == -1:
   random_number = random.randint(0, len(ip_list) - 1)
  ip_info = ip_list[random_number]
  ip_url_next = '://' + ip_info['address'] + ':' + ip_info['port']
  proxies = {'http': 'http' + ip_url_next, 'https': 'https' + ip_url_next}
  return random_number, proxies


# 程序主入口
if __name__ == '__main__':
 """只是爬取了书籍的第一页,按照评价排序"""
 start_time = datetime.datetime.now()
 url = 'https://book.douban.com/tag/""" 整理分析数据 """
  sub_type_list = []
  a = table.find_all('a')
  for book_type in a:
   sub_type_list.append(book_type.text)
  article_type_list.append(sub_type_list)

 for sub in article_type_list:
  for sub1 in sub:
   title = '==============' + sub1 + '=============='
   print(title)
   print(base_url + sub1 + '"text-align: center">

总结


使用这样简单的代理IP,基本上就可以应付在爬爬爬着被封IP的情况了.而且没有使用自己的IP,间接的保护?!?!
大家有其他的更加快捷的方法，欢迎大家可以拿出来交流和讨论，谢谢。
好了，以上就是这篇文章的全部内容了，希望本文的内容对大家的学习或者工作具有一定的参考学习价值，谢谢大家对的支持。