Sample usage for chunk¶
Chunking¶
>>> from nltk.chunk import *
>>> from nltk.chunk.util import *
>>> from nltk.chunk.regexp import *
>>> from nltk import Tree
>>> tagged_text = "[ The/DT cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] [ the/DT dog/NN ] chewed/VBD ./."
>>> gold_chunked_text = tagstr2tree(tagged_text)
>>> unchunked_text = gold_chunked_text.flatten()
Chunking uses a special regexp syntax for rules that delimit the chunks. These rules must be converted to ‘regular’ regular expressions before a sentence can be chunked.
>>> tag_pattern = "<DT>?<JJ>*<NN.*>"
>>> regexp_pattern = tag_pattern2re_pattern(tag_pattern)
>>> regexp_pattern
import sysfrom PyQt6.QtWidgets import QApplication, QMainWindow, QPushButton, QLineEdit, QLabel, QVBoxLayout, QWidgetfrom process_text import process_textclass MainWindow(QMainWindow): def __init__(self): super().__init__() # Set the main window's properties self.setWindowTitle("Text Processor") self.setGeometry(100, 100, 400, 200) # x, y, width, height # Set up the layout layout = QVBoxLayout() # Create widgets self.input_text = QLineEdit() self.input_text.setPlaceholderText("Enter your text here...") self.process_button = QPushButton("Process Text") self.result_label = QLabel("Result will appear here...") # Add widgets to the layout layout.addWidget(self.input_text) layout.addWidget(self.process_button) layout.addWidget(self.result_label) # Set the central widget and layout central_widget = QWidget() central_widget.setLayout(layout) self.setCentralWidget(central_widget) # Connect the button's click signal to the function that processes the text self.process_button.clicked.connect(self.on_process_button_clicked) def on_process_button_clicked(self): # Get the text from the input field input_text = self.input_text.text() # Process the text using the function defined earlier result_text = process_text(input_text) # Display the result in the label self.result_label.setText(result_text)# Create and run the applicationapp = QApplication(sys.argv)window = MainWindow()window.show()sys.exit(app.exec())
Construct some new chunking rules.
>>> chunk_rule = ChunkRule(r"<.*>+", "Chunk everything")
>>> strip_rule = StripRule(r"<VBD|IN|\.>", "Strip on verbs/prepositions")
>>> split_rule = SplitRule("<DT><NN>", "<DT><NN>",
... "Split successive determiner/noun pairs")
Create and score a series of chunk parsers, successively more complex.
>>> chunk_parser = RegexpChunkParser([chunk_rule], chunk_label='NP')
>>> chunked_text = chunk_parser.parse(unchunked_text)
>>> print(chunked_text)
(S
(NP
The/DT
cat/NN
sat/VBD
on/IN
the/DT
mat/NN
the/DT
dog/NN
chewed/VBD
./.))
>>> chunkscore = ChunkScore()
>>> chunkscore.score(gold_chunked_text, chunked_text)
>>> print(chunkscore.precision())
0.0
>>> print(chunkscore.recall())
0.0
>>> print(chunkscore.f_measure())
0
>>> for chunk in sorted(chunkscore.missed()): print(chunk)
(NP The/DT cat/NN)
(NP the/DT dog/NN)
(NP the/DT mat/NN)
>>> for chunk in chunkscore.incorrect(): print(chunk)
(NP
The/DT
cat/NN
sat/VBD
on/IN
the/DT
mat/NN
the/DT
dog/NN
chewed/VBD
./.)
>>> chunk_parser = RegexpChunkParser([chunk_rule, strip_rule],
... chunk_label='NP')
>>> chunked_text = chunk_parser.parse(unchunked_text)
>>> print(chunked_text)
(S
(NP The/DT cat/NN)
sat/VBD
on/IN
(NP the/DT mat/NN the/DT dog/NN)
chewed/VBD
./.)
>>> assert chunked_text == chunk_parser.parse(list(unchunked_text))
>>> chunkscore = ChunkScore()
>>> chunkscore.score(gold_chunked_text, chunked_text)
>>> chunkscore.precision()
0.5
>>> print(chunkscore.recall())
0.33333333...
>>> print(chunkscore.f_measure())
0.4
>>> for chunk in sorted(chunkscore.missed()): print(chunk)
(NP the/DT dog/NN)
(NP the/DT mat/NN)
>>> for chunk in chunkscore.incorrect(): print(chunk)
(NP the/DT mat/NN the/DT dog/NN)
>>> chunk_parser = RegexpChunkParser([chunk_rule, strip_rule, split_rule],
... chunk_label='NP')
>>> chunked_text = chunk_parser.parse(unchunked_text, trace=True)
# Input:
<DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>
# Chunk everything:
{<DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>}
# Strip on verbs/prepositions:
{<DT> <NN>} <VBD> <IN> {<DT> <NN> <DT> <NN>} <VBD> <.>
# Split successive determiner/noun pairs:
{<DT> <NN>} <VBD> <IN> {<DT> <NN>}{<DT> <NN>} <VBD> <.>
>>> print(chunked_text)
(S
(NP The/DT cat/NN)
sat/VBD
on/IN
(NP the/DT mat/NN)
(NP the/DT dog/NN)
chewed/VBD
./.)
>>> chunkscore = ChunkScore()
>>> chunkscore.score(gold_chunked_text, chunked_text)
>>> chunkscore.precision()
1.0
>>> chunkscore.recall()
1.0
>>> chunkscore.f_measure()
1.0
>>> chunkscore.missed()
[]
>>> chunkscore.incorrect()
[]
>>> chunk_parser.rules()
[<ChunkRule: '<.*>+'>, <StripRule: '<VBD|IN|\\.>'>,
<SplitRule: '<DT><NN>', '<DT><NN>'>]
Printing parsers:
>>> print(repr(chunk_parser))
<RegexpChunkParser with 3 rules>
>>> print(chunk_parser)
RegexpChunkParser with 3 rules:
Chunk everything
<ChunkRule: '<.*>+'>
Strip on verbs/prepositions
<StripRule: '<VBD|IN|\\.>'>
Split successive determiner/noun pairs
<SplitRule: '<DT><NN>', '<DT><NN>'>
Regression Tests¶
ChunkParserI¶
ChunkParserI is an abstract interface – it is not meant to be instantiated directly.
>>> ChunkParserI().parse([])
Traceback (most recent call last):
. . .
NotImplementedError
ChunkString¶
ChunkString can be built from a tree of tagged tuples, a tree of trees, or a mixed list of both:
>>> t1 = Tree('S', [('w%d' % i, 't%d' % i) for i in range(10)])
>>> t2 = Tree('S', [Tree('t0', []), Tree('t1', ['c1'])])
>>> t3 = Tree('S', [('w0', 't0'), Tree('t1', ['c1'])])
>>> ChunkString(t1)
<ChunkString: '<t0><t1><t2><t3><t4><t5><t6><t7><t8><t9>'>
>>> ChunkString(t2)
<ChunkString: '<t0><t1>'>
>>> ChunkString(t3)
<ChunkString: '<t0><t1>'>
Other values generate an error:
>>> ChunkString(Tree('S', ['x']))
Traceback (most recent call last):
. . .
ValueError: chunk structures must contain tagged tokens or trees
The str() for a chunk string adds spaces to it, which makes it line up with str() output for other chunk strings over the same underlying input.
>>> cs = ChunkString(t1)
>>> print(cs)
<t0> <t1> <t2> <t3> <t4> <t5> <t6> <t7> <t8> <t9>
>>> cs.xform('<t3>', '{<t3>}')
>>> print(cs)
<t0> <t1> <t2> {<t3>} <t4> <t5> <t6> <t7> <t8> <t9>
The _verify() method makes sure that our transforms don’t corrupt the chunk string. By setting debug_level=2, _verify() will be called at the end of every call to xform.
>>> cs = ChunkString(t1, debug_level=3)
>>> # tag not marked with <...>:
>>> cs.xform('<t3>', 't3')
Traceback (most recent call last):
. . .
ValueError: Transformation generated invalid chunkstring:
<t0><t1><t2>t3<t4><t5><t6><t7><t8><t9>
>>> # brackets not balanced:
>>> cs.xform('<t3>', '{<t3>')
Traceback (most recent call last):
. . .
ValueError: Transformation generated invalid chunkstring:
<t0><t1><t2>{<t3><t4><t5><t6><t7><t8><t9>
>>> # nested brackets:
>>> cs.xform('<t3><t4><t5>', '{<t3>{<t4>}<t5>}')
Traceback (most recent call last):
. . .
ValueError: Transformation generated invalid chunkstring:
<t0><t1><t2>{<t3>{<t4>}<t5>}<t6><t7><t8><t9>
>>> # modified tags:
>>> cs.xform('<t3>', '<t9>')
Traceback (most recent call last):
. . .
ValueError: Transformation generated invalid chunkstring: tag changed
>>> # added tags:
>>> cs.xform('<t9>', '<t9><t10>')
Traceback (most recent call last):
. . .
ValueError: Transformation generated invalid chunkstring: tag changed
Chunking Rules¶
Test the different rule constructors & __repr__ methods:
>>> r1 = RegexpChunkRule('<a|b>'+ChunkString.IN_STRIP_PATTERN,
... '{<a|b>}', 'chunk <a> and <b>')
>>> r2 = RegexpChunkRule(re.compile('<a|b>'+ChunkString.IN_STRIP_PATTERN),
... '{<a|b>}', 'chunk <a> and <b>')
>>> r3 = ChunkRule('<a|b>', 'chunk <a> and <b>')
>>> r4 = StripRule('<a|b>', 'strip <a> and <b>')
>>> r5 = UnChunkRule('<a|b>', 'unchunk <a> and <b>')
>>> r6 = MergeRule('<a>', '<b>', 'merge <a> w/ <b>')
>>> r7 = SplitRule('<a>', '<b>', 'split <a> from <b>')
>>> r8 = ExpandLeftRule('<a>', '<b>', 'expand left <a> <b>')
>>> r9 = ExpandRightRule('<a>', '<b>', 'expand right <a> <b>')
>>> for rule in r1, r2, r3, r4, r5, r6, r7, r8, r9:
... print(rule)
<RegexpChunkRule: '<a|b>(?=[^\\}]*(\\{|$))'->'{<a|b>}'>
<RegexpChunkRule: '<a|b>(?=[^\\}]*(\\{|$))'->'{<a|b>}'>
<ChunkRule: '<a|b>'>
<StripRule: '<a|b>'>
<UnChunkRule: '<a|b>'>
<MergeRule: '<a>', '<b>'>
<SplitRule: '<a>', '<b>'>
<ExpandLeftRule: '<a>', '<b>'>
<ExpandRightRule: '<a>', '<b>'>
tag_pattern2re_pattern() complains if the tag pattern looks problematic:
>>> tag_pattern2re_pattern('{}')
Traceback (most recent call last):
. . .
ValueError: Bad tag pattern: '{}'
RegexpChunkParser¶
A warning is printed when parsing an empty sentence:
>>> parser = RegexpChunkParser([ChunkRule('<a>', '')])
>>> parser.parse(Tree('S', []))
Warning: parsing empty text
Tree('S', [])
RegexpParser¶
>>> parser = RegexpParser('''
... NP: {<DT>? <JJ>* <NN>*} # NP
... P: {<IN>} # Preposition
... V: {<V.*>} # Verb
... PP: {<P> <NP>} # PP -> P NP
... VP: {<V> <NP|PP>*} # VP -> V (NP|PP)*
... ''')
>>> print(repr(parser))
<chunk.RegexpParser with 5 stages>
>>> print(parser)
chunk.RegexpParser with 5 stages:
RegexpChunkParser with 1 rules:
NP <ChunkRule: '<DT>? <JJ>* <NN>*'>
RegexpChunkParser with 1 rules:
Preposition <ChunkRule: '<IN>'>
RegexpChunkParser with 1 rules:
Verb <ChunkRule: '<V.*>'>
RegexpChunkParser with 1 rules:
PP -> P NP <ChunkRule: '<P> <NP>'>
RegexpChunkParser with 1 rules:
VP -> V (NP|PP)* <ChunkRule: '<V> <NP|PP>*'>
>>> print(parser.parse(unchunked_text, trace=True))
# Input:
<DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>
# NP:
{<DT> <NN>} <VBD> <IN> {<DT> <NN>}{<DT> <NN>} <VBD> <.>
# Input:
<NP> <VBD> <IN> <NP> <NP> <VBD> <.>
# Preposition:
<NP> <VBD> {<IN>} <NP> <NP> <VBD> <.>
# Input:
<NP> <VBD> <P> <NP> <NP> <VBD> <.>
# Verb:
<NP> {<VBD>} <P> <NP> <NP> {<VBD>} <.>
# Input:
<NP> <V> <P> <NP> <NP> <V> <.>
# PP -> P NP:
<NP> <V> {<P> <NP>} <NP> <V> <.>
# Input:
<NP> <V> <PP> <NP> <V> <.>
# VP -> V (NP|PP)*:
<NP> {<V> <PP> <NP>}{<V>} <.>
(S
(NP The/DT cat/NN)
(VP
(V sat/VBD)
(PP (P on/IN) (NP the/DT mat/NN))
(NP the/DT dog/NN))
(VP (V chewed/VBD))
./.)
Test parsing of other rule types:
>>> print(RegexpParser('''
... X:
... }<a><b>{ # strip rule
... <a>}{<b> # split rule
... <a>{}<b> # merge rule
... <a>{<b>}<c> # chunk rule w/ context
... '''))
chunk.RegexpParser with 1 stages:
RegexpChunkParser with 4 rules:
strip rule <StripRule: '<a><b>'>
split rule <SplitRule: '<a>', '<b>'>
merge rule <MergeRule: '<a>', '<b>'>
chunk rule w/ context <ChunkRuleWithContext: '<a>', '<b>', '<c>'>
Illegal patterns give an error message:
>>> print(RegexpParser('X: {<foo>} {<bar>}'))
Traceback (most recent call last):
. . .
ValueError: Illegal chunk pattern: {<foo>} {<bar>}