import osimport queueimport threadingimport timefrom dataclasses import dataclassfrom concurrent.futures import ThreadPoolExecutor, as_completedimport tkinter as tkfrom tkinter import ttk, messagebox, filedialog, scrolledtextimport requestsfrom bs4 import BeautifulSoupimport pandas as pd# ==========================================# 1. 界面统一配置对象# ==========================================@dataclassclass UIConfig: font_normal: tuple = ("Microsoft YaHei UI", 10) font_title: tuple = ("Microsoft YaHei UI", 12, "bold") font_log: tuple = ("Consolas", 9) entry_width: int = 24 combo_width: int = 22 padx: int = 8 pady: int = 6 button_pady: int = 8 label_fg: str = "#222222" status_ok: str = "#2f855a" status_warn: str = "#dd6b20" status_error: str = "#c53030"# ==========================================# 2. 基础控件与组合控件构建函数# ==========================================def create_label(factory, parent, text, row, column=0, **kwargs): options = {"text": text, "font": factory.config.font_normal, "fg": factory.config.label_fg} options.update(kwargs) widget = tk.Label(parent, **options) widget.grid(row=row, column=column, padx=factory.config.padx, pady=factory.config.pady, sticky="w") return factory.remember(widget)def create_entry(factory, parent, row, column=1, textvariable=None, width=None, **kwargs): options = { "font": factory.config.font_normal, "width": width or factory.config.entry_width, "textvariable": textvariable, } options.update(kwargs) widget = tk.Entry(parent, **options) widget.grid(row=row, column=column, padx=factory.config.padx, pady=factory.config.pady, sticky="ew") return factory.remember(widget)def create_button(factory, parent, text, command, row, column=0, columnspan=1): widget = ttk.Button(parent, text=text, command=command) widget.grid( row=row, column=column, columnspan=columnspan, padx=factory.config.padx, pady=factory.config.button_pady, sticky="ew", ) return factory.remember(widget)def create_labeled_entry(factory, parent, label_text, row, variable=None, unit_text=""): factory.create("label", parent, text=label_text, row=row, column=0) entry = factory.create("entry", parent, row=row, column=1, textvariable=variable) if unit_text: factory.create("label", parent, text=unit_text, row=row, column=2) return entrydef create_spinbox_row(factory, parent, label_text, row, variable, from_=1, to=100, step=1): factory.create("label", parent, text=label_text, row=row, column=0) spinbox = tk.Spinbox( parent, from_=from_, to=to, increment=step, textvariable=variable, width=factory.config.entry_width, font=factory.config.font_normal, ) spinbox.grid(row=row, column=1, padx=factory.config.padx, pady=factory.config.pady, sticky="ew") return factory.remember(spinbox)def create_path_picker(factory, parent, label_text, row, variable): """自定义控件:路径选择输入框 + 浏览按钮""" factory.create("label", parent, text=label_text, row=row, column=0) entry = factory.create("entry", parent, row=row, column=1, textvariable=variable) def browse(): path = filedialog.asksaveasfilename( defaultextension=".xlsx", filetypes=[("Excel Files", "*.xlsx"), ("All Files", "*.*")], ) if path: variable.set(path) btn = ttk.Button(parent, text="浏览...", width=8, command=browse) btn.grid(row=row, column=2, padx=factory.config.padx, pady=factory.config.pady) factory.remember(btn) return entrydef create_status_badge(factory, parent, textvariable, level="ok"): color_map = { "ok": factory.config.status_ok, "warn": factory.config.status_warn, "error": factory.config.status_error, } widget = tk.Label( parent, textvariable=textvariable, font=factory.config.font_normal, fg="white", bg=color_map.get(level, factory.config.status_ok), padx=10, pady=4, ) widget.pack(anchor="w", padx=factory.config.padx * 3, pady=factory.config.pady) return factory.remember(widget)def create_log_panel(factory, parent, height=8): """自定义控件:实时日志展示面板""" log_area = scrolledtext.ScrolledText( parent, height=height, font=factory.config.font_log, state="disabled", bg="#1e1e1e", fg="#d4d4d4" ) log_area.pack(fill="both", expand=True, padx=factory.config.padx * 2, pady=factory.config.pady) return log_area# ==========================================# 3. 升级版注册式控件工厂类# ==========================================class RegistryWidgetFactory: """扩展版注册式 Tkinter 工厂,支持装饰器注册与动态控制""" def __init__(self, config: UIConfig): self.config = config self._creators = {} self._widgets = [] self.register_defaults() def register(self, widget_type: str, creator=None): """支持函数调用 register("type", fn) 或装饰器 @register("type")""" if creator is None: def decorator(fn): if not callable(fn): raise TypeError("creator 必须是可调用对象") self._creators[widget_type] = fn return fn return decorator if not callable(creator): raise TypeError("creator 必须是可调用对象") self._creators[widget_type] = creator def create(self, widget_type: str, parent, **kwargs): creator = self._creators.get(widget_type) if creator is None: available = ", ".join(sorted(self._creators.keys())) raise KeyError(f"未注册控件类型: {widget_type},可用类型: {available}") return creator(factory=self, parent=parent, **kwargs) def remember(self, widget): self._widgets.append(widget) return widget def set_all_state(self, state: str): """批量设置控件状态(如抓取过程中禁用按钮/输入框)""" for widget in self._widgets: try: if hasattr(widget, "configure"): widget.configure(state=state) except tk.TclError: pass def register_defaults(self): """注册内置基础控件""" self.register("label", create_label) self.register("entry", create_entry) self.register("button", create_button) self.register("labeled_entry", create_labeled_entry) self.register("spinbox_row", create_spinbox_row) self.register("path_picker", create_path_picker) self.register("status_badge", create_status_badge) self.register("log_panel", create_log_panel)# ==========================================# 4. 并发爬虫逻辑(ThreadPoolExecutor)# ==========================================def fetch_single_page(page_number: int): """单页采集任务""" url = f"https://gz.lianjia.com/ershoufang/pg{page_number}/" headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/120.0.0.0 Safari/537.36" ) } try: response = requests.get(url, headers=headers, timeout=10) if response.status_code != 200: return page_number, [], f"请求失败,HTTP 状态码: {response.status_code}" soup = BeautifulSoup(response.text, "html.parser") rows = [] for house_info in soup.find_all("li", {"class": "clear LOGVIEWDATA LOGCLICKDATA"}): row = {} pos = house_info.find("div", {"class": "positionInfo"}) house = house_info.find("div", {"class": "houseInfo"}) follow = house_info.find("div", {"class": "followInfo"}) unit_p = house_info.find("div", {"class": "unitPrice"}) total_p = house_info.find("div", {"class": "priceInfo"}) row["区域"] = pos.get_text(strip=True) if pos else "" row["房型"] = house.get_text(strip=True) if house else "" row["关注"] = follow.get_text(strip=True) if follow else "" row["单价"] = unit_p.get_text(strip=True) if unit_p else "" row["总价"] = total_p.get_text(strip=True) if total_p else "" rows.append(row) return page_number, rows, None except Exception as e: return page_number, [], str(e)# ==========================================# 5. 主应用界面与多线程控制器# ==========================================class LianjiaScraperApp: def __init__(self, root): self.root = root self.root.title("链家二手房并发采集器 (注册式工厂版)") self.root.geometry("680x520") # 初始化工厂与变量 self.factory = RegistryWidgetFactory(UIConfig()) self.msg_queue = queue.Queue() default_save_path = os.path.join(os.path.expanduser("~"), "Desktop", "lianjia_data.xlsx") self.start_page_var = tk.StringVar(value="1") self.end_page_var = tk.StringVar(value="5") self.workers_var = tk.StringVar(value="4") self.path_var = tk.StringVar(value=default_save_path) self.status_var = tk.StringVar(value="状态:就绪") self.build_ui() self.poll_queue() # 开启定时拉取队列日志 def build_ui(self): title = tk.Label(self.root, text="广州链家二手房数据并发采集面板", font=self.factory.config.font_title) title.pack(pady=(12, 4)) # 参数配置表单 form = ttk.Frame(self.root) form.pack(fill="x", padx=20, pady=5) form.columnconfigure(1, weight=1) self.factory.create("spinbox_row", form, label_text="起始页码:", row=0, variable=self.start_page_var, from_=1, to=100) self.factory.create("spinbox_row", form, label_text="结束页码:", row=1, variable=self.end_page_var, from_=1, to=100) self.factory.create("spinbox_row", form, label_text="并发线程数:", row=2, variable=self.workers_var, from_=1, to=16) self.factory.create("path_picker", form, label_text="保存文件路径:", row=3, variable=self.path_var) # 按钮栏 btn_frame = ttk.Frame(self.root) btn_frame.pack(fill="x", padx=20, pady=5) btn_frame.columnconfigure(0, weight=1) btn_frame.columnconfigure(1, weight=1) self.factory.create("button", btn_frame, text="启动并发抓取", command=self.start_crawling, row=0, column=0) self.factory.create("button", btn_frame, text="重置默认参数", command=self.reset_params, row=0, column=1) # 状态栏与日志面板 self.factory.create("status_badge", self.root, textvariable=self.status_var, level="ok") self.log_area = self.factory.create("log_panel", self.root, height=10) def log(self, message: str): """将日志放入队列""" self.msg_queue.put(("LOG", message)) def poll_queue(self): """主线程定时轮询日志队列,避免 UI 阻塞""" while not self.msg_queue.empty(): msg_type, content = self.msg_queue.get() if msg_type == "LOG": self.log_area.configure(state="normal") self.log_area.insert(tk.END, f"[{time.strftime('%H:%M:%S')}] {content}\n") self.log_area.see(tk.END) self.log_area.configure(state="disabled") elif msg_type == "STATUS": self.status_var.set(content) elif msg_type == "FINISH": self.factory.set_all_state("normal") messagebox.showinfo("完成", content) self.root.after(100, self.poll_queue) def start_crawling(self): """启动后台抓取线程""" try: start_p = int(self.start_page_var.get()) end_p = int(self.end_page_var.get()) workers = int(self.workers_var.get()) save_path = self.path_var.get().strip() if start_p > end_p: raise ValueError("起始页码不能大于结束页码!") if not save_path: raise ValueError("保存路径不能为空!") except ValueError as err: messagebox.showwarning("参数错误", str(err)) return # 锁定界面控件 self.factory.set_all_state("disabled") self.status_var.set("状态:数据采集并发运行中...") # 启动后台线程执行采集 threading.Thread( target=self._async_task, args=(start_p, end_p, workers, save_path), daemon=True, ).start() def _async_task(self, start_p: int, end_p: int, workers: int, save_path: str): """在后台线程中调度 ThreadPoolExecutor 进行并发爬取""" all_data = [] total_pages = end_p - start_p + 1 completed_count = 0 self.log(f"开始抓取 Task:页码 {start_p} ~ {end_p},线程数 {workers}") # 使用 ThreadPoolExecutor 进行并发 with ThreadPoolExecutor(max_workers=workers) as executor: future_to_page = { executor.submit(fetch_single_page, page): page for page in range(start_p, end_p + 1) } for future in as_completed(future_to_page): page_num, page_data, err = future.result() completed_count += 1 if err: self.log(f"第 {page_num} 页抓取失败: {err}") else: all_data.extend(page_data) self.log(f"第 {page_num} 页成功抓取 {len(page_data)} 条记录 ({completed_count}/{total_pages})") # 保存 Excel if all_data: try: df = pd.DataFrame(all_data) df.to_excel(save_path, index=False) self.log(f"数据已成功保存至: {save_path}") self.msg_queue.put(("STATUS", "状态:采集完成")) self.msg_queue.put(("FINISH", f"成功抓取 {len(all_data)} 条数据,已保存至:\n{save_path}")) except Exception as e: self.log(f"保存 Excel 异常: {e}") self.msg_queue.put(("STATUS", "状态:保存数据失败")) self.msg_queue.put(("FINISH", f"保存 Excel 失败: {e}")) else: self.log("未采集到任何有效数据!") self.msg_queue.put(("STATUS", "状态:未获取到数据")) self.msg_queue.put(("FINISH", "未抓取到任何数据,请检查网络或请求限制。")) def reset_params(self): default_save_path = os.path.join(os.path.expanduser("~"), "Desktop", "lianjia_data.xlsx") self.start_page_var.set("1") self.end_page_var.set("5") self.workers_var.set("4") self.path_var.set(default_save_path) self.status_var.set("状态:参数已重置")if __name__ == "__main__": root = tk.Tk() app = LianjiaScraperApp(root) root.mainloop()