def crawl():
    # 初始 API: https://api.hahow.in/api/courses?limit=12&status=PUBLISHED
    # 接續 API: https://api.hahow.in/api/courses?latestId=54d5a117065a7e0e00725ac0&latestValue=2015-03-27T15:38:27.187Z&limit=30&status=PUBLISHED
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                             'AppleWebKit/537.36 (KHTML, like Gecko) '
                             'Chrome/59.0.3071.115 Safari/537.36'}
    url = 'https://api.hahow.in/api/courses'
    courses = list()
    resp_courses = requests.get(url + '?limit=30&status=PUBLISHED', headers=headers).json()
    while resp_courses:  # 有回傳資料則繼續下一輪擷取
        time.sleep(3)  # 放慢爬蟲速度
        courses += resp_courses
        param = '?latestId={0}&latestValue={1}&limit=30&status=PUBLISHED'.format(
            courses[-1]['_id'], courses[-1]['incubateTime'])
        resp_courses = requests.get(url + param, headers=headers).json()
	# 將課程資料存下來後續分析使用
    with open('hahow_courses.json', 'w', encoding='utf-8') as f:
        json.dump(courses, f, indent=2, sort_keys=True, ensure_ascii=False)
    return courses

with open('hahow_courses.json', 'r', encoding='utf-8') as f:
    courses = json.load(f)
	
# 取出程式類課程的募資價/上線價/學生數，並顯示統計資料
pre_order_prices = list()
prices = list()
tickets = list()
lengths = list()
for c in courses:
    if '55de81ac9d1fa51000f94770' in c['categories']:
        pre_order_prices.append(c['preOrderedPrice'])
        prices.append(c['price'])
        tickets.append(c['numSoldTickets'])
        lengths.append(c['totalVideoLengthInSeconds'])
		
print('程式類課程共有 %d 堂' % len(prices))  # 23
print('平均募資價:', np.mean(pre_order_prices))  # 719.09
print('平均上線價:', np.mean(prices))  # 1322.57
print('平均學生數:', np.mean(tickets))  # 483.22
print('平均課程分鐘:', np.mean(lengths)/60)  # 515.12

corrcoef = np.corrcoef([tickets, pre_order_prices, prices, lengths])
print('募資價與學生數之相關係數: ', corrcoef[0, 1])  # 0.18
print('上線價與學生數之相關係數: ', corrcoef[0, 2])  # 0.36
print('課程長度與學生數之相關係數: ', corrcoef[0, 3])  # 0.65

with open('example.json', 'r', encoding='utf-8') as f:
    data_list = json.load(f)
    images = []
    pushes = []
    for d in data_list:
        images.append(d['num_image'])
        pushes.append(d['push_count'])

print('圖片數:', images, 'Max:', max(images), 'Min:', min(images))
print('推文數:', pushes, 'Max:', max(pushes), 'Min:', min(pushes))

# 圖片數: [3, 7, 1, 12, 9, 1, 2, 13, 0, 5, 27, 5, 1, 8, 0, 1, 14, 2, 3, 2, 1, 25, 3, 14, 27, 2] Max: 27 Min: 0
# 推文數: [18, 20, 0, 0, 3, 6, 2, 12, 1, 13, 11, 5, 0, 20, 1, 7, 6, 2, 2, 0, 0, 32, 10, 13, 9, 2] Max: 32 Min: 0

def mean(x):
    return sum(x) / len(x)

print('平均圖片數:', mean(images), '平均推文數:', mean(pushes))

# 平均圖片數: 7.230769230769231 平均推文數: 7.5

def de_mean(x):
    x_bar = mean(x)
    return [x_i - x_bar for x_i in x]


def variance(x):
    deviations = de_mean(x)
    variance_x = 0
    for d in deviations:
        variance_x += d**2
    variance_x /= len(x)
    return variance_x


def dot(x, y):
    dot_product = sum(v_i * w_i for v_i, w_i in zip(x, y))
    dot_product /= (len(x))
    return dot_product

def correlation(x, y):
    variance_x = variance(x)
    variance_y = variance(y)
    sd_x = math.sqrt(variance_x)
    sd_y = math.sqrt(variance_y)
    dot_xy = dot(de_mean(x), de_mean(y))
    return dot_xy/(sd_x*sd_y)

print('相關係數:', correlation(images, pushes))

# 相關係數: 0.5258449106844523

def decile(num):  # 將數字十分位化
    return (num // 10) * 10

from collections import Counter
histogram = Counter(decile(push) for push in pushes)
print(histogram)
# Counter({0: 17, 10: 6, 20: 2, 30: 1})

# 畫出 histogram
from matplotlib import pyplot as plt

plt.bar([x-4 for x in histogram.keys()], histogram.values(), 8)
plt.axis([-5, 35, 0, 20])
plt.title('Pushes')
plt.xlabel('# of pushes')
plt.ylabel('# of posts')
plt.xticks([10 * i for i in range(4)])
plt.show()

plt.scatter(images, pushes)
plt.title('# of image v.s. push')
plt.xlabel('# of image')
plt.ylabel('# of push')
plt.axis('equal')
plt.show()

articles = [
    {'push_count': 8, 'title': '[正妹] 韓國瑜的女兒', 'href': '/bbs/Beauty/M.1482411674.A.855.html'}
    {'push_count': 5, 'title': '[正妹] 佐々木優佳里 Happiness', 'href': '/bbs/Beauty/M.1482414319.A.C09.html'}
    {'push_count': 13, 'title': '[正妹] 甜美笑容 第一二五彈', 'href': '/bbs/Beauty/M.1482416491.A.656.html'}
    ...
]

PTT_URL = 'https://www.ptt.cc'

for article in articles:
    page = get_web_page(PTT_URL + article['href'])
    if page:
        img_urls = parse(page)
        save(img_urls, article['title'])
        article['num_image'] = len(img_urls)        

def parse(dom):
    soup = BeautifulSoup(dom, 'html.parser')
    links = soup.find(id='main-content').find_all('a')
    img_urls = []
    for link in links:
        if link['href'].startswith('http://i.imgur.com'):
            img_urls.append(link['href'])
    return img_urls

test_urls = [
    'http://i.imgur.com/A2wmlqW.jpg',
    'http://i.imgur.com/A2wmlqW',  # 沒有 .jpg
    'https://i.imgur.com/A2wmlqW.jpg',
    'http://imgur.com/A2wmlqW.jpg',
    'https://imgur.com/A2wmlqW.jpg',
    'https://imgur.com/A2wmlqW',
    'http://m.imgur.com/A2wmlqW.jpg',
    'https://m.imgur.com/A2wmlqW.jpg'
]

import re
for url in test_urls:
    print(re.match('^https?://(i.)?(m.)?imgur.com', url))  # 符合則回傳 SRE_Match Object, 不符合則回傳 None
# <_sre.SRE_Match object; span=(0, 18), match='http://i.imgur.com'>
# <_sre.SRE_Match object; span=(0, 18), match='http://i.imgur.com'>
# <_sre.SRE_Match object; span=(0, 19), match='https://i.imgur.com'>
# <_sre.SRE_Match object; span=(0, 16), match='http://imgur.com'>
# <_sre.SRE_Match object; span=(0, 17), match='https://imgur.com'>
# <_sre.SRE_Match object; span=(0, 17), match='https://imgur.com'>
# <_sre.SRE_Match object; span=(0, 18), match='http://m.imgur.com'>
# <_sre.SRE_Match object; span=(0, 19), match='https://m.imgur.com'>

def parse(dom):
    soup = BeautifulSoup(dom, 'html.parser')
    links = soup.find(id='main-content').find_all('a')
    img_urls = []
    for link in links:
        if re.match(r'^https?://(i.)?(m.)?imgur.com', link['href']):
            img_urls.append(link['href'])
    return img_urls

def save(img_urls, title):
    if img_urls:
        try:
            dname = title.strip()  # 用 strip() 去除字串前後的空白
            os.makedirs(dname)
            for img_url in img_urls:
                if img_url.split('//')[1].startswith('m.'):
                    img_url = img_url.replace('//m.', '//i.')
                if not img_url.split('//')[1].startswith('i.'):
                    img_url = img_url.split('//')[0] + '//i.' + img_url.split('//')[1]
                if not img_url.endswith('.jpg'):
                    img_url += '.jpg'
                fname = img_url.split('/')[-1]
                urllib.request.urlretrieve(img_url, os.path.join(dname, fname))
        except Exception as e:
            print(e)

import json

with open('data.json', 'w', encoding='utf-8') as f:
    json.dump(articles, f, indent=2, sort_keys=True, ensure_ascii=False)

html_doc = """
<html>
  <head>
    <title>我是網頁標題</title>
    <style>
    .large {
      color:blue;
      text-align: center;
    }
    </style>
  </head>
  <body>
    <h1 class="large">我是變色且置中的抬頭</h1>
    <p id="p1">我是段落一</p>
    <p id="p2" style="">我是段落二</p>
    <div><a href='http://blog.castman.net' style="font-size:200%;">我是放大的超連結</a></div>
  </body>
</html>
"""

from bs4 import BeautifulSoup
soup = BeautifulSoup(html_doc, 'html.parser')
print(soup)
# <html>
# <head>
# <title>我是網頁標題</title>
# <style>
# .large {
#   color:blue;
#   text-align: center;
# }
# </style>
# </head>
# <body>
# <h1 class="large" style="">我是變色且置中的抬頭</h1>
# <p id="p1">我是段落一</p>
# <p id="p2" style="">我是段落二</p>
# <div><a href="http://blog.castman.net" style="font-size:200%;">我是放大的超連結</a></div>
# </body>
# </html>

soup.find('p')            # 回傳第一個被 <p> </p> 所包圍的區塊
# <p id="p1">我是段落一</p>

soup.find('p', id='p2')   # 回傳第一個被 <p> </p> 所包圍的區塊且 id="p2"
# <p id="p2" style="">我是段落二</p>

soup.find(id='p2')        # 回傳第一個 id="p2" 的區塊
# <p id="p2" style="">我是段落二</p>

soup.find_all('p')        # 回傳所有被 <p> </p> 所包圍的區塊
# [<p id="p1">我是段落一</p>, <p id="p2" style="">我是段落二</p>]

soup.find('h1', 'large')  # 找尋第一個 <h1> 區塊且 class="large"
# <h1 class="large" style="">我是變色且置中的抬頭</h1>

paragraphs = soup.find_all('p')
for p in paragraphs:
    print(p['id'], p.text)
# p1 我是段落一
# p2 我是段落二

a = soup.find('a')
print(a['href'], a['style'], a.text)
# http://blog.castman.net font-size:200%; 我是放大的超連結

print(soup.find('h1')['class'])  # 因為 class 可以有多個值，故回傳 list
# ['large']

def get_articles(dom, date):
    soup = BeautifulSoup(dom, 'html.parser')

    articles = []  # 儲存取得的文章資料
    divs = soup.find_all('div', 'r-ent')
    for d in divs:
        if d.find('div', 'date').string == date:  # 發文日期正確
            # 取得推文數
            push_count = 0
            if d.find('div', 'nrec').string:
                try:
                    push_count = int(d.find('div', 'nrec').string)  # 轉換字串為數字
                except ValueError:  # 若轉換失敗，不做任何事，push_count 保持為 0
                    pass

            # 取得文章連結及標題			
            if d.find('a'):  # 有超連結，表示文章存在，未被刪除
                href = d.find('a')['href']
                title = d.find('a').string
                articles.append({
                    'title': title,
                    'href': href,
                    'push_count': push_count
                })
    return articles

page = get_web_page('https://www.ptt.cc/bbs/Beauty/index.html')
if page:
    date = time.strftime("%m/%d").lstrip('0')  # 今天日期, 去掉開頭的 '0' 以符合 PTT 網站格式
    current_articles = get_articles(page, date)
    for post in current_articles:
        print(post)
# {'push_count': 8, 'title': '[正妹] 韓國瑜的女兒', 'href': '/bbs/Beauty/M.1482411674.A.855.html'}
# {'push_count': 5, 'title': '[正妹] 佐々木優佳里 Happiness', 'href': '/bbs/Beauty/M.1482414319.A.C09.html'}
# {'push_count': 13, 'title': '[正妹] 甜美笑容 第一二五彈', 'href': '/bbs/Beauty/M.1482416491.A.656.html'}
# {'push_count': 0, 'title': '[帥哥] 佐々木小次郎', 'href': '/bbs/Beauty/M.1482417495.A.733.html'}
# {'push_count': 14, 'title': '[正妹] 短髮有可愛 FB女孩Round 255', 'href': '/bbs/Beauty/M.1482419748.A.D25.html'}
# {'push_count': 7, 'title': '[正妹] 筧美和子', 'href': '/bbs/Beauty/M.1482419973.A.32C.html'}
# {'push_count': 58, 'title': '[正妹] 教你公民好嗎', 'href': '/bbs/Beauty/M.1482420690.A.AE7.html'}
# {'push_count': 9, 'title': '[正妹] 熟女了', 'href': '/bbs/Beauty/M.1482420814.A.021.html'}
# {'push_count': 10, 'title': '[正妹] 佐々木琴子', 'href': '/bbs/Beauty/M.1482421163.A.C42.html'}
# {'push_count': 0, 'title': '[正妹] 野生日本賽車女神', 'href': '/bbs/Beauty/M.1482421895.A.F6B.html'}

def get_web_page(url):
    resp = requests.get(
        url=url,
        cookies={'over18': '1'}
    )
    if resp.status_code != 200:
        print('Invalid url:', resp.url)
        return None
    else:
        return resp.text

page = get_web_page('https://www.ptt.cc/bbs/Beauty/index.html')
if page:
	print(page)