有没有合适的PYTHON抓字典的案例可参考？

stiggg

懂点PYTHON,想学学前人怎么爬字典的。目前只看到BT4BAIDU大大的开源，大大的代码很典范，可是代码四五年了，似乎网站也改版，具体细节的作用比较难揣摩了。不知道还有其它的比较好可参考学习？

毛小驴

顶起

你去哪里

我也想知道这个。

你去哪里

我也想知道这个。

scirem

1. #!/usr/bin/env python3
   3 o+ p, Y) c' R1 t6 i- K
2. # -*- coding: utf-8 -*-
   
3. 
   
4. import os
   
5. import requests
   1 A0 ?7 ]; }' S6 @$ |- h+ c
6. import re
   2 a9 z2 G0 j% ]7 a2 P# Q8 q
7. import time
   # ~4 b4 ]/ \+ W) O5 F+ W4 D, q: F
8. import sqlite3
   0 y- s; M, X; O* K7 O
9. 
   
10. from string import ascii_lowercase
    
11. 
    
12. from pathlib import Path
    
13. from urllib.parse import urljoin
    + n4 c; R4 {0 x, S( {+ t& z3 U
14. from html.parser import HTMLParser
    
15. from bs4 import BeautifulSoup
    % b# Y. @" ^- ?' e5 ?8 @
16. 
    
17. webster_url = "https://www.merriam-webster.com/browse/dictionary/"
    
18. oxford_url = "https://www.oxfordlearnersdictionaries.com/browse/english/"
    
19. macmillan_url = "https://www.macmillandictionary.com/browse/british/"
    
20. cambridge_url ="https://dictionary.cambridge.org/browse/english/"
    0 z4 C7 ~7 n1 @) e; ?
21. 
    1 ?, ~* A6 u; z' X5 y- }
22. cambridge_english_chinese_url = "https://dictionary.cambridge.org/browse/english-chinese-simplified/"
    " G4 _5 j. Y$ q! a7 l
23. 
    
24. base_url = "https://dictionary.cambridge.org/search/direct/"
    
25. header = {"User-Agent": "Chrome/87.0.4280.88 Safari/537.36"}
    + v$ z: E8 U: O7 q
26. payload = {'datasetsearch': 'english-chinese-simplified', 'q': ""}
    
27. 
    ! `4 H0 Z) W0 |' S" Z* S
28. conn = sqlite3.connect("Cambridge Advanced Learner's English-Chinese Dictionary.db")
    
29. 
    1 ?. z8 n$ Y  |
30. conn.execute("""
    ; `' U* O7 C8 ~8 r$ u0 O  N
31. CREATE TABLE IF NOT EXISTS cambridge (
    - K0 `$ `! ?% n: z/ [1 h
32.     entry TEXT PRIMARY KEY
    6 p& N4 }- D# p# o1 C* E% L- s3 |
33.                UNIQUE
    ; v- Q) @; x# R1 K3 t6 P2 T( D, X
34.                NOT NULL,
    
35.     url   TEXT NOT NULL,
    
36.     html  TEXT NOT NULL,
    
37.     latex TEXT
    
38. );
    
39. """)
    
40. 
    & K2 c3 c( F' n& b- \
41. conn.commit()
    6 B1 S* A  s. c- g1 {' X0 s
42. 
    $ g. X0 |  S* h% h( W6 b& Z5 p
43. 
    
44. def latexify(string):
    
45.     result = ""
    + z1 I6 J3 e5 v" M" p3 [, H
46.     trimmed_string = re.sub(r"\s{2,}", " ", string)
    
47.     for char in re.sub(r"\s+([^\w'"({[])", r"\1", trimmed_string):
    2 [2 o# h. l/ O+ H
48.         if char == "%":
    7 J! e9 ~1 r4 L# o% r2 @
49.             result += r"\%"
    
50.         elif char == "\":
    
51.             result += r"\textbackslash{}"
    
52.         elif char == "$":
    
53.             result += r"\$"
    + }  T* n, I- z8 {8 K
54.         elif char == "#":
    
55.             result += r"\#"
    
56.         elif char == "&":
    
57.             result += r"\&"
    
58.         elif char == "{":
    ! i3 s. d2 _' x* `$ k
59.             result += r"\{"
    
60.         elif char == "}":
    
61.             result += r"\}"
    / P8 v  A- o4 B2 h3 S& W1 v6 G( g& g
62.         elif char == "^":
    5 D" I7 W4 y8 M/ W' F2 K+ b3 ^+ d
63.             result += r"\^"
    
64.         elif char == "_":
      X4 Q  P# w" G( C( x2 K4 f
65.             result += r"\_"
    
66.         elif char == "~":
    2 r# s* v  v$ L+ E$ e. i
67.             result += "r\textasciitilde{}"
    
68.         else:
    4 P+ w, f$ p, x' Q2 e
69.             result += char
    / w' [  W  z8 b" {) I  ]; l
70.     return result
    8 z( X/ d  M4 z0 L% Z7 K- v$ K
71. 
    - j' Q1 f# x7 S" v+ {, @
72. def latexify_html(beautifulsoup_object):
    
73.     try:
    $ c4 n9 Q+ n! S, ]& _) K
74.         return latexify(re.sub(r"\s{2,}", " ", beautifulsoup_object.get_text().replace("\n", " ")).strip())
    8 s* m) c$ U+ W; v' g2 ^0 r
75.     except:
    ( ]  k3 u$ M" i) ]; ?6 ?2 _
76.         return latexify(beautifulsoup_object)
    ) |4 p& i2 @! _! F" z
77. 
    
78. 
    5 f  ~4 s' O$ h1 `0 _
79. class CambridgeDictionaryScraper:
    ) y" h$ T9 ~2 D) N, b! k
80.     """ Scraper for Cambridge Dictionary """
    2 y& D0 U: A; E" B  O
81.     url_set = set() ## Shared by all instance
    
82. 
    ( d, S1 M" a$ S& |
83.     def __init__(self):
    
84.         for item in conn.execute("SELECT url FROM cambridge;"):
    
85.             self.url_set.add(item[0])
    1 l& g) `# b6 H
86.         print("Already Downloaded " + str(len(self.url_set)) + " Words!")
    ; `9 O  X6 a4 g
87. 
    
88. 
    & }( N$ I+ y1 ]4 i
89.     def __del__(self):
    1 @! m+ h: v7 P7 J7 D6 K9 D0 ~
90.         conn.commit()
    
91. 
    
92. 
    
93.     def get_word_page(self, url):
    
94.         r = requests.get(url, headers=header)        
    # j+ {2 e) ^# E& i* U" {
95.         bs_obj = BeautifulSoup(r.text, "html.parser")
    # w( C2 {( _4 n8 h+ t8 `# f
96.         entry_tag = bs_obj.find("div", {"class": "entry"})
    
97.         if not entry_tag:
    " J! C( U* f2 T0 U! S
98.             entry_tag = bs_obj.find("div", {"class": "di-body"})
    % T  O3 w$ V. j2 V1 x, c
99.         if not entry_tag:
    
100.             ## Beta Words
     
101.             entry_tag = bs_obj.find("div", {"id": "entryContent"})
     $ _: i  b9 |! L' a+ T9 h
102.         if not entry_tag:
     . F0 g& j, p5 \& A& X: x5 I$ t
103.             entry_tag = bs_obj
     % Z5 L. \- l- S8 }: M5 z( Z
104. 
     
105.         if not entry_tag:
     
106.             entry_tag = bs_obj
     
107.         
     ' m6 f& A( r. w
108.         for tag in entry_tag.find_all("script"):
     
109.             tag.extract()
     5 x* B$ G# d& w0 }4 J
110.             
     5 u6 A! O% H9 N' |5 p( Y- W
111.         result_string = str(entry_tag)
     
112.         return result_string
     
113. 
     5 U% |7 \) ^1 f  [' X& R
114.    
     9 U& w" ]& P- \  N  y5 Y) E
115.     def start(self, url):
     
116.         r = requests.get(url, headers=header)
     0 b9 j/ u4 k& H$ }0 O1 k
117. 
     # J7 Q( N  d2 r, e+ T3 i4 D; N- r
118.         bs_obj = BeautifulSoup(r.text, "html.parser")
     . H4 X9 G& g3 s
119. 
     . u: G/ r* M4 i
120.         for li_tag in bs_obj.select("li.lpr-10"):
     
121.             child_url = urljoin(cambridge_url, li_tag.a.attrs["href"])
     
122.             print(child_url)
     
123.             self.find_child_entry(child_url)
     $ j. b* s, Y/ k6 ]$ B1 Y
124. 
     " |% {& \0 H7 ]& c8 l
125. 
     ' v; v) D9 P1 N" l
126.     def find_child_entry(self, url):
     - \% E  a7 ~$ O+ a
127.         r = requests.get(url, headers=header)
     
128.         bs_obj = BeautifulSoup(r.text, "html.parser")
     
129.         for li_tag in bs_obj.select("li.t-i"):
     * h9 a2 |: b7 D2 `% d/ ~- \3 A
130.             child_url = urljoin(url, li_tag.a.attrs["href"]).strip()
     
131.             child_text = li_tag.get_text().strip()
     0 _$ n: I8 N. s3 g
132.             if "..." in child_text:
     6 k6 s) k9 v( B* ^: ]: e
133.                 self.find_child_entry(child_url)
     ) ~1 s0 z6 H1 ?8 A5 T4 r
134.             else:
     9 ?# X( a; H, w" C0 ~2 J" E7 i/ b
135.                 if child_url in self.url_set:
     0 z- Z, e! i: K  z
136.                     continue
     
137.                 print(child_text + "\t" + child_url)
     
138.                 conn.execute("INSERT INTO cambridge (entry, url, html) VALUES (?, ?, ?);",
     
139.                 (child_text, child_url, self.get_word_page(child_url)))
     
140.                 conn.commit()
     " W/ n. w4 A/ l2 i2 ]# P% j
141.                 self.url_set.add(child_url)
     
142. 
     
143. 
     . Q$ D# {  h3 ?# I0 ^( @$ k1 v5 e
144. class CambridgeDictionaryExtractor():
     
145.     def __init__(self, entry, entry_html = ""):
     : T7 \. n! v2 o( q4 G+ ?3 ?
146.         self.entry = latexify(entry)
     
147.         self.entry_html = entry_html
     
148.         self.result = ""
     
149. 
     
150.     # def __del__(self):
     
151.     #     pass
     
152. 
     ' `6 @4 @  t8 H0 n) \; S0 P" I
153. 
     ! v" Y$ K; R: Z# N2 U8 ^
154.     def extract(self):
     2 A7 y# h$ c0 R3 z. E0 s
155.         """
     
156.         <div class="pr idiom-block">
     
157.             <div class="idiom-block"></div>
     
158.         </div>
     5 o0 m- x. R8 `. q. I' `% @& v
159.         """
     . p" W- H# j! N5 N  @* t/ _' t
160.         bs_obj = BeautifulSoup(self.entry_html, "html.parser")
     , F6 |1 h) s4 E: K
161.         self.result += "\\begin{entry}{" + self.entry + "}"
     
162.         for part_of_speech_tag in bs_obj.find_all("div", {"class": "entry-body__el"}):
     3 k% U0 _' H3 p$ T9 L
163.             self.result += "\n\\begin{Partofspeech}\n" + self.process_part_of_speech(part_of_speech_tag) + "\n\\end{Partofspeech}"
     2 J& {- o& s1 p/ i
164.         idiom_block = bs_obj.find("div", {"class": "idiom-block"})
     
165.         if idiom_block:
     
166.             for idiom_block in idiom_block.find_all("div", {"class": "idiom-block"}):
     5 z0 @0 X' q+ Q, w# f- J
167.                 self.result += "\n\\begin{idiom}" + self.process_idiom_block(idiom_block) + "\n\\end{idiom}"
     
168.         self.result += "\n\\end{entry}\n\n"
     
169. 
     
170.    
     0 S3 o6 U( V! T, w
171.     def process_idiom_block(self, idiom_block):
     # ]+ U! k; U  O7 W* K1 z/ k2 [  A
172.         result = ""
     
173.         idiom_body = idiom_block.find("span", {"class": "idiom-body"})
     $ \  ?6 M# W! ]8 t: t
174.         if idiom_body:
     
175.             for sense_tag in idiom_body.find_all("div", {"class": "dsense"}):
     + U3 k! N& n! S/ g0 ]$ e: `* d
176.                 result += self.process_sense_tag(sense_tag)
     
177.         return result
     + w0 G5 {7 e1 k+ E+ a1 {
178. 
     ) o$ ^; k% x5 h! s$ c8 Z" r
179.    
     % Y3 x: h" z0 e# _/ A
180. 
     8 P# S9 L, i( S4 u; D
181.     def get_smart_vocabulary(self, smart_vocabulary_tag):
     & G' ^1 t4 O$ s  E$ m
182.         result = ""
     / u9 e7 E% ?) x$ K$ l9 }& M# [1 k; }
183.         for li_tag in smart_vocabulary_tag.find_all("li"):
     
184.             result += "\\smart{" + latexify_html(li_tag) + "}\n"
     
185.         return "\n\\begin{smartvocabulary}\n" + result + "\\end{smartvocabulary}\n"
     7 c1 J  D, O( \. P3 a9 Y* d
186. 
     
187. 
     
188.     def process_part_of_speech(self, part_of_speech_tag):
     
189.         """
     2 M4 s6 [# u1 `( C4 C
190.         <div class="entry-body__el">
     , j6 w0 X/ N( e( F$ K% D6 y
191.             <div class="pos-header"></div>
     / k$ _( M2 S% N% m5 h8 [
192.             <div class="pos-body"></div>
     
193.             <div class="pr relativDiv"></div>
     8 g9 Q2 G7 ^  \
194.         <div>
     
195.         """
     - e* `3 c: c1 g* z# [% Y7 r- e
196.         result = ""
     & R' L+ N  O3 ~
197.         header_tag = part_of_speech_tag.find("div", {"class": "pos-header"})
     
198.         if header_tag:
     1 v# q! g# E( y
199.             result += self.process_part_of_speech_header(header_tag)
     
200.         body_tag = part_of_speech_tag.find("div", {"class": "pos-body"})
     
201.         if body_tag:
     8 r' C- c$ u2 ^
202.             result += self.process_part_of_speech_body(body_tag)
     
203.         pv_block_tag = part_of_speech_tag.find("div", {"class": "pv-block"})
     
204.         if pv_block_tag:
     
205.             result += self.process_pv_block(pv_block_tag)
     0 p* a6 }2 N! X; i
206.         return result.strip()
     % {, J: ?+ o! x0 Q+ a2 V8 Z
207. 
     ( A4 n, M9 n5 S0 y$ K
208.    
     
209.     def process_pv_block(self, tag):
     4 |* `( `. p& f5 c" m: v
210.         """
     
211.         <div class="pv-block">
     
212.             <div class="di-title"></div>
     $ w3 e8 I" D4 k6 [8 P; m- O
213.             <span clss="di-info"></span>
     ( y" b- F4 l1 B+ v5 t0 F0 J
214.             <span class="pv-body dpv-body">
     - ^0 h- a! D  L% c* g
215.                 <div class="pr dsense dsense-noh">
     
216.             <span>
     ( y' q9 z, J/ K8 `- ?* J
217.         <div>
     
218.         """
     ) K* \' Z7 q& f0 f1 M
219.         result = ""
     
220.         for item in tag.find_all("div",{"class", "sense-body"}):
     & k2 C) U9 A, o
221.             result += self.process_sense_body(item)
     ' g) ]* p3 z& {1 L7 w. f
222.         return result
     
223. 
     ! q0 ~3 a& B) v4 _
224. 
     
225.    
     
226.     def process_part_of_speech_header(self, header_tag):
     
227.         result = ""
     
228.         # title_tag = header_tag.find("div", {"class": "di-title"})
     
229.         # if title_tag:
     
230.         #     result += process_header_title(title_tag)
     4 j1 Q3 V) N& H, v' G$ G! o
231.         posgram_tag = header_tag.find("div", {"class": "posgram"})
     , P' S- t' e# u1 o/ P5 u8 w
232.         if posgram_tag:
     
233.             result += self.process_part_of_speech_grammar(posgram_tag)
     * f: r# I9 N* d; a% e( ^! m
234.         for pronunciation_tag in header_tag.find_all("span", {"class": "dpron-i"}):
     0 z3 [9 V8 a2 j, ?' ?
235.             result += self.process_pronunciation(pronunciation_tag)
     
236. 
     7 k- F2 N( |7 [( R% a
237.         return result.strip()
     
238. 
     - V' m$ N8 d+ w+ z0 c
239. 
     $ Z: y3 @$ U- P. n: f9 N; R- F
240.     def process_header_title(self, title_tag):
     
241.         ## <span class="hw dhw">record</span>
     
242.         result = ""
     
243.         headword_tag = title_tag.find("span", {"class": "hw"})
     
244.         if headword_tag:
     
245.             result += "\\entry{" + latexify_html(headword_tag) + "}\n"
     
246.         else:
     9 w, U& g0 J' _, s
247.             result += "\\entry{" + latexify_html(title_tag) + "}\n"
     
248.         return result
     
249. 
     9 `/ c+ s, `2 K- r
250.     def process_part_of_speech_grammar(self, posgram_tag):
     
251.         result = ""
     
252.         part_of_speech_tag = posgram_tag.find("span", {"class": "pos"})
     
253.         if part_of_speech_tag:
     # ]2 U1 K/ d6 H
254.             result += "\n\\pos{" + latexify_html(part_of_speech_tag) + "}"
     
255.         gram_tag = posgram_tag.find("span", {"class": "gc"})
     # r* s0 ]! `9 q  N
256.         if gram_tag:
     2 O; @5 c; Z+ g8 N
257.             result += "\n\\posgram{" + latexify_html(gram_tag)  + "}"
     
258.         return result
     
259. 
     / b4 V: J- r% S" a1 D! h( j2 m6 O
260.     def process_pronunciation(self, pronunciation_tag):
     8 R: ^: x4 n3 U% r- G
261.         is_us_pronunciation = False
     0 D/ W/ p' g+ ^, A+ f3 G5 r, ^
262.         if "us" in pronunciation_tag.attrs["class"]:
     
263.             is_us_pronunciation = True
     ' c8 o; z2 d  v0 E+ ~# A
264.         result = ""
     
265.         audio_tag = pronunciation_tag.find("source", {"type": "audio/mpeg"})
     
266.         
     + V3 x6 V* W; [9 w" G; {/ H' I
267.         ipa_tag = pronunciation_tag.find("span", {"class": "ipa"})
     6 Z( _. {8 m) O9 K; e- O( _
268.         if ipa_tag:
     
269.             if is_us_pronunciation:
     
270.                 result += "\n\\ipaus{" + latexify_html(ipa_tag) + "}"
     
271.             else:
     
272.                 result += "\n\\ipauk{" + latexify_html(ipa_tag) + "}"
     % t$ G/ H' L. g
273.         if audio_tag:
     
274.             audio_url = urljoin("https://dictionary.cambridge.org/", audio_tag.attrs["src"])
     $ f* n0 z, z) I- D1 ]& R2 p1 ~
275.             result += "\n\pronuniation{" + audio_url + "}"
     : B8 o& u7 q4 s: N- r2 P9 n7 W
276.         return result
     
277. 
       [& B2 B( X/ C" Q9 j7 S
278. 
     7 I4 v$ z/ T8 M) y8 Z
279. 
     4 k. e9 `* r8 P) c- x* d7 Q" y2 v
280.     def process_sense_head(self, tag):
     
281.         text = latexify_html(tag)
     ; r" E6 O6 a: v, X
282.         if "(" in text:
     
283.             left_bracket_index = text.index("(")
     ! z8 p2 o! v& O* ?$ t" [% @
284.         else:
     4 H# F( Z4 y5 X# M. X& ]
285.             left_bracket_index = 0
     2 m' @& Z+ G& J) y
286.         if ")" in text:
     - a: e, `* X" L$ X: O/ J
287.             right_bracket_index = text.index(")")
     
288.         else:
     2 a, |1 Z" f+ ^" U5 `6 z9 ]
289.             right_bracket_index = len(text)
     
290.         return "\n\\shortmeaning{" + text[left_bracket_index + 1:  right_bracket_index].strip() + "}"
     ( E9 s* z1 Z+ I! N' t* V# ^
291. 
     
292. 
     
293.     def get_definition(self, tag):
     $ }; F/ Z+ ^* j
294.         result = ""
     : d' h1 O. J& ~
295.         for def_info_tag in tag.select("span.def-into"):
     
296.             result += latexify_html(def_info_tag)
     7 F) R; T1 s; O& E4 V. E1 D
297.         for def_tag in tag.select("div.def"):
     ' e+ Q9 N" f9 w
298.             result += latexify_html(def_tag)
     $ A6 [3 O% B2 o4 h: m. m
299.         return result
     - ^' d0 Q! {; p
300. 
     7 H$ H( N  V7 w) x$ ?6 ]
301. 
     
302.     def process_def_block(self, tag):
     2 O. F. ^( F( W& a3 n
303.         result = ""
     
304.         def_tag = tag.find("div", {"class": "ddef_h"})
     
305.         if def_tag:
     ; ^/ Q( R7 _, W( F1 c# }
306.             result += "\n\\meaningen{" + self.get_definition(def_tag) + "}"
     
307.         try:
     
308.             def_trans_tag = tag.select("span.trans")[0]
     * i( _7 w2 [- y6 c
309.             if def_trans_tag:
     
310.                 result += "\n\\meaningcn{" + latexify_html(def_trans_tag) + "}"
     / l2 M/ e! c  E# J
311.         except:
     % z2 h% j6 }- P3 I( M/ S8 ]
312.             pass
     2 D" C( T' q5 u
313.         for example_tag in tag.select("span.eg"):
     ' W- O7 g) A+ T6 W# s8 k) y
314.             result += "\n\\example{" + latexify_html(example_tag) + "}"
     
315.         return result
     
316. 
     4 M# F; {+ W, k# q) N) W/ }  ?# n
317. 
       o. O% Q+ ]0 |( n% y
318.     def process_phrase_block(self, phrase_block_tag):
     . O4 h% |8 I5 V, Y7 m
319.         """
     1 P6 a0 n9 }; @$ q: l$ s
320.         <div class="phrase-head dphrase_h">...</div>
     
321.         <div class="phrase-body dphrase_b">...</div>
     
322.         <div class="bb hax">...</div>
     % [+ H; C; Y: q0 Y6 n% ]
323.         """
     
324.         result = "\\begin{phrase}{"
     
325.         result += phrase_block_tag.select("span.phrase-title")[0].get_text().strip() + "}"
     5 S0 O5 c+ P1 `4 _
326.         return result + "\\end{pharse}\n"
     
327. 
     
328.     def process_sense_body(self, tag):
     
329.         """
     
330.         <div class="pr phrase-block dphrase-block">...</div>
     ! t- x3 r9 d+ u, h2 d2 c1 Q
331.         <div class="def-block ddef_block">...</div>
     
332.         <div class="bb hax">...</div>
     # I, m& W7 r& v- R2 h1 M8 G5 ?. l
333.         """
     ) f' {. a! U4 j( ~& g
334.         result = ""
     / A8 M' j8 V, h) B. `6 d
335.         for def_block in tag.select("div.def-block"):
     + ]+ h. g) z4 C" P9 N! \6 V  |, U
336.             result += self.process_def_block(def_block)
     , w+ i+ @5 `. q; w  O
337.         for phrase_block in tag.select("div.pharse-block"):
       Z+ Z% b0 a  g( s3 D& o6 u( R# s
338.             result += self.process_phrase_block(phrase_block)
     7 q2 l$ I7 V( f
339.         return result
     
340.         
     
341.     def process_sense_tag(self, sense_tag):
       O2 g+ V" i" {2 p# N0 }
342.         """
     
343.         <h3 class="dsense_h">...</h3>
     , ^: W' E, `% {+ V6 q' e: n
344.         <div class="sense-body dsense_b">...</div>
     
345.         <div class="smartt daccord">...</div>           # Smart Vocabulary
     0 U$ A1 Q8 D3 g2 f
346.         <div class="bb hax">...</div>
     
347.         """
     9 ?9 D  ?4 z2 x0 A
348.         result = ""
     8 B5 F7 L5 c: Z7 z4 o# w/ v! }0 @
349.         sense_head_tag = sense_tag.find("h3", {"class": "dsense_h"})
     
350.         if sense_head_tag:
     2 s; A) ]. F+ y  B/ p9 `3 W& M* T. s
351.             result += self.process_sense_head(sense_head_tag)
     
352.         for sense_body_tag in sense_tag.select("div.sense-body"):
     # w( @/ S$ j3 e8 b6 B8 b* m
353.             result += self.process_sense_body(sense_body_tag)
     2 Z1 _  n# E. b, M  l
354.         for smart_vocabulary_tag in sense_tag.find_all("div", {"class": "smartt"}):
     & j! M6 J! G  Q5 I' j. ]# b
355.             result += self.get_smart_vocabulary(smart_vocabulary_tag)
     
356.             
     
357.         return result
     
358. 
     % u- Q) [6 z/ P" l7 w$ V2 L4 t; q
359.     def process_part_of_speech_body(self, body_tag):
     4 w' _5 O% q) v, l% H4 o
360.         """
     
361.         <div class="pr dsense">...</div>
     9 |3 {0 z7 N' C, _. @: z
362.         <div class="pr dsense">...</div>
     
363.         """
     
364.         result = ""
     
365.         for sense_tag in body_tag.select("div.dsense"):
     ' \6 @. y/ h( A6 j9 i
366.             result += self.process_sense_tag(sense_tag)
     $ L  D0 }* c1 Y1 Z, b: d7 H! J) q
367.         return result
     
368.    
     ' u' w3 S* Q- O3 \5 u. ^' M2 a7 n  a9 X
369. 
     
370. if __name__ == "__main__":
     
371.     string = ""
     
372.     CambridgeDictionaryScraper().start(cambridge_english_chinese_url)
     
373.     for row in conn.execute("SELECT entry, url, html FROM cambridge;"):
     
374.         entry = row[0]
     $ R- U8 j+ i2 G' C- X
375.         print(entry)
     0 V  i: [: E1 h# O; H3 ^% d
376.         url = row[1]
     
377.         html = row[2]
     
378.         record = CambridgeDictionaryExtractor(entry, entry_html=html)
       f) x, U2 e7 W( h, E( N
379.         record.extract()
     - n  ^& z& H8 b4 [. F# J, |
380.         string += record.result + "\n"
     6 O$ p/ M' [* B' E) U
381.         #print(record.result)
     
382. 
     
383.     with open("./final.tex", "w", encoding="utf-8") as f:
     
384.         try:
     3 p; x: S1 d8 T) ?6 x; F* t0 T8 P( C
385.             for char in ascii_lowercase:
     , e& g9 \. f1 H: N8 R) ~& ~8 n
386.                 string = string.replace("\\begin{entry}{" + char, "\\section{" + char + "}\n\n\\begin{entry}{" + char, 1)
     : j% B7 A  S8 X1 I
387.         except:
     & O6 j, \& y, ~9 m+ p; m9 b) ?% V
388.             pass
     ' T& ?: v; g. f/ }
389.         
     
390.         f.write(string.replace("", ""))
     $ W* o- `$ @# G) B% t7 O( e) T! r% Q
391. 
     $ @6 `, m- S4 i! o
392. 
     
393.         

复制代码