API bot 만들기 - 뉴스스크랩

jkim.coding · 1월 31, 2025, 6:46오전

여기서 뉴스 열개 스크랩해서 오려고 했는데

스크랩하니
https://news.google.com/read/CBMiXkFVX3lxTE44Z3FzY2NtbEZhVFdjR0ZrU1NBRFU2TUZPbGg1Rmx6aTNkOXpUQ0VpV3hVOEdFRVVfUHdjcnhkTEY1bUVSM3hkdjBHSURjOWwtSHdtZ3BScWtsdmgtMVHSAXZBVV95cUxNNGM3bXRQOVhkaUJhRDQ0NDRuTXR5TTRBT0FjZ1c0V0VZMFhqRzFwWEpJU3piRU9HeXV1YThGSS11V19qX0p1SGpHLTZHM1Q1TDhRRTY0bFVFV1F3TzZEQkhBYU9PUHdlSW16dDhMOUdINlQtRm9R?hl=ko&gl=KR&ceid=KR%3Ako
링크
이런식으로 redirect로 되어있어서
막혔습니다

이거 우회하려고

r = requests.get('https://youtu.be/dQw4w9WgXcQ') 
print(r.url) # https://www.youtube.com/watch?v=dQw4w9WgXcQ&feature=youtu.be

이거도 안되고

from selenium import webdriver

def get_final_link_js(url):
    # Requires selenium and a suitable driver, e.g. ChromeDriver
    driver = webdriver.Chrome()
    try:
        driver.get(url)
        # Wait or drive the page as needed
        final_url = driver.current_url
    finally:
        driver.quit()
    return final_url

이방법은 API Function 스크립트로 안되고 인스턴스 만든다음 해야할거 같아서 일단 보류

import urllib.request
import re
import urllib.parse

def extract_read_links(url, limit=10):
    """
    Fetch the HTML from a Google News URL, find links that start with "./read/",
    convert them into full URLs (https://news.google.com/read/...), and return
    up to `limit` such links.
    """
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req) as response:
        html_content = response.read().decode("utf-8", errors="ignore")

    # Regex to capture: href="./read/..."
    pattern = re.compile(r'href="\./read/[^"]+"')
    matches = pattern.findall(html_content)

    results = []
    for match in matches:
        # match looks like: href="./read/CBMiSkF..."
        relative_url = match[6:-1]  # remove href=" and trailing quote
        # "./read/..." => "https://news.google.com/read/..."
        full_url = "https://news.google.com/" + relative_url[2:]
        results.append(full_url)
        if len(results) == limit:
            break

    return results

def get_final_destination(url):
    """
    Follow HTTP 3xx redirects automatically.
    If the final URL never changes, look for a <meta http-equiv="refresh"> tag
    in the HTML. If there's no server-side 3xx redirect or meta-refresh,
    return the original URL.
    """
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req) as resp:
        resolved_url = resp.geturl()
        if resolved_url != url:
            # Found a standard 3xx redirect
            return resolved_url

        html = resp.read().decode("utf-8", errors="ignore")

    # Check for <meta http-equiv="refresh" content="0; url=...">
    match = re.search(
        r'<meta[^>]+http-equiv\s*=\s*"refresh"[^>]+url\s*=\s*([^";]+)',
        html,
        flags=re.IGNORECASE
    )
    if match:
        meta_url = match.group(1).strip()
        return urllib.parse.urljoin(url, meta_url)

    return url

def main():
    # Example Google News URL (Bellevue/Seattle news, etc.)
    google_news_url = (
        "https://news.google.com/topics/"
        "CAAqHAgKIhZDQklTQ2pvSWJHOWpZV3hmZGpJb0FBUAE/"
        "sections/CAQiTkNCSVNORG9JYkc5allXeGZkakpDRUd4dlkyRnNYM1l5"
        "WDNObFkzUnBiMjV5Q2hJSUwyMHZNR1E1YW5KNkNnb0lMMjB2TUdRNWFuSW9BQS"
        "owCAAqLAgKIiZDQklTRmpvSWJHOWpZV3hmZGpKNkNnb0lMMjB2TUdRNWFuSW9BQVABUAE"
        "?hl=ko&gl=KR&ceid=KR%3Ako"
    )

    # Step 1: Extract up to 10 ./read/... links
    read_links = extract_read_links(google_news_url, limit=10)
    if not read_links:
        print("No ./read/ links found.")
        return

    # Step 2: For each extracted link, find its ultimate final destination
    for i, link in enumerate(read_links, start=1):
        final_url = get_final_destination(link)
        print(f"{i}. Extracted: {link}\n   Final: {final_url}\n")

if __name__ == "__main__":
    main()

jkim.coding · 2월 1, 2025, 5:28오전

import urllib.request
from bs4 import BeautifulSoup
import json
from urllib.parse import urljoin

def parse_articles(html):
    soup = BeautifulSoup(html, 'html.parser')
    
    articles = []
    
    for element in soup.find_all(attrs={"data-article-source-name": True}):
        source = element.get('data-article-source-name')
        
        link = element.find('a')
        
        if link:
            articles.append({
                "source": source,
                "text" : element.text,
                "url": link.get('href'),
                "img": element.find('img').get('src')
            })
    return articles

def get_html():
    url = "https://www.google.com/finance/?hl=ko&gl=US&source=news"
    req = urllib.request.Request(
        url,
        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}
    )
    with urllib.request.urlopen(req) as res:
        return res.read().decode('utf-8')

def main():
    html = get_html()
    
    results = parse_articles(html)
    print(results)
    print(json.dumps(results, ensure_ascii=False, indent=2))

if __name__ == "__main__":
    main()

일단 여기까지 끝

jkim.coding · 2월 1, 2025, 7:14오전

html value 가 매번 바뀌어서 2초씩 기다리면서 1분동안 30번 리퀘스트
find soup.find_all(attrs={"data-article-source-name": True}) 찾아서 거기에 있는 아티클 긁어온뒤 출력
href 주소 넣으면 자동으로 변환되기 때문에 raw_text로 변환만하면 끝

매번 html 에 data-article-source-name 디버깅하는게 tricky

import urllib.request
import time
from bs4 import BeautifulSoup
import json
from datetime import datetime
import pytz

# Target website
API_KEY = ''  # Replace with actual API key
POST_URL = 'https://kwork.life/posts.json'
API_USERNAME = ''
POST_CATEGORY = 19

# Constants
GOOGLE_FINANCE_URL = "https://www.google.com/finance/?hl=ko&gl=US&source=news"
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'

TIMEZONE = 'America/Los_Angeles'

def parse_articles(soup):
    articles = []

    for element in soup.find_all(attrs={"data-article-source-name": True}):
        source = element.get('data-article-source-name')
        
        link = element.find('a')
        
        if link:
            articles.append({
                "source": source,
                "text": element.text.strip(),
                "url": link.get('href'),
                "img": element.find('img').get('src') if element.find('img') else None
            })
    return articles

def get_html():
    req = urllib.request.Request(
        GOOGLE_FINANCE_URL,
        headers={'User-Agent': USER_AGENT}
    )
    with urllib.request.urlopen(req) as res:
        return res.read().decode('utf-8')

def send_post_request(json_results):
    headers = {
        'Content-Type': 'application/json',
        'Api-Key': API_KEY,
        'Api-Username': API_USERNAME
    }
    
    # Create title with PST time
    pst = datetime.now(pytz.timezone(TIMEZONE))
    title = f"{pst.strftime('%m-%d')} 오늘자 주요 미국소식을 다룬 뉴스기사 {len(json_results)}가지 입니다"
    raw_text = "오늘자 미국 주요 뉴스를 다룬 한국 기사들 입니다\n"
    
    for article in json_results:
        raw_text += article['url'] + "\n\n"

    body = {
        'category': POST_CATEGORY,
        'title': title,
        'raw': raw_text
    }
    print(body)
    if len(json_results) == 0:
        return None
    
    req = urllib.request.Request(
        POST_URL,
        data=json.dumps(body).encode('utf-8'),
        headers=headers,
        method='POST'
    )
    
    try:
        with urllib.request.urlopen(req) as res:
            response_data = res.read().decode('utf-8')
            print("Post successfully sent!")
            print(response_data)
            return response_data
    except urllib.error.HTTPError as e:
        print(f"HTTP Error: {e.code} - {e.reason}")
        print(e.read().decode())  # Print server response for debugging
        return None
    except Exception as e:
        print(f"An error occurred: {e}")
        return None

def main():
    html = get_html()
    soup = BeautifulSoup(html, 'html.parser')  # Parse HTML with BeautifulSoup
    
    # Check if no elements with the attribute "data-article-source-name" are found
    if len(soup.find_all(attrs={"data-article-source-name": True})) == 0:
        time.sleep(2)  # Wait for 2 seconds
        print("html fail ->sleep 2sec")
        main()  # Retry by calling main again
        return  # Ensure this call exits after recursion

    results = parse_articles(soup)  # Pass parsed soup, not raw HTML
    
    # Print results for debugging purposes
    print(json.dumps(results, ensure_ascii=False, indent=2))
    
    # Send POST request with parsed articles
    send_post_request(results)

if __name__ == "__main__":
    main()

jkim.coding · 2월 1, 2025, 7:14오전

이게 결과물인데 내일부터 정상으로 올라오면 성공