BeautifulSoup 用于具有不同模板的多个 URL

如果我理解正确，您想基于以下内容创建新数据框page_url_df：

import requests
import pandas as pd
from bs4 import BeautifulSoup


page_url1 = "https://www.vet.upenn.edu/research/centers-laboratories/research-laboratory/research-laboratory/anguera-laboratory"
page_url2 = (
    "https://www.med.upenn.edu/apps/faculty/index.php/g20001100/p8866286"
)
page_url_lst = {"url": [page_url1, page_url2], "template": [1, 2]}
page_url_df = pd.DataFrame(page_url_lst)


def get_template_1(url):
    data = []
    soup = BeautifulSoup(requests.get(url).content, "lxml")
    for e in soup.select("#tabs-publications em"):
        data.append(
            {
                "author": e.previous.get_text(strip=True)[:-1],
                "title": e.get_text(strip=True),
                "journal": e.next_sibling.get_text(strip=True),
                "source": url,
            }
        )
    return data


def get_template_2(url):
    data = []
    soup = BeautifulSoup(requests.get(url).text, "lxml")
    for a in soup.find_all("span", {"class": "fac_citation"}):
        data.append(
            {
                "author": a.find("b").get_text(),
                "title": a.find("i").get_text(strip=True),
                "journal": a.find("i").next_sibling.get_text(strip=True),
                "source": url,
            }
        )
    return data


all_data = []
for _, row in page_url_df.iterrows():
    print("Getting", row["url"])
    if row["template"] == 1:
        all_data.extend(get_template_1(row["url"]))
    elif row["template"] == 2:
        all_data.extend(get_template_2(row["url"]))


df_out = pd.DataFrame(all_data)

# print sample data
print(df_out.head().to_markdown())

印刷：