Text Segmentation Using Exponential Models

@misc{citeulike:596223, abstract = {This paper introduces a new statistical approach to partitioning text automatically into coherent segments. Our approach enlists both short-range and long-range language models to help it sniff out likely sites of topic changes in text. To aid its search, the system consults a set of simple lexical hints it has learned to associate with the presence of boundaries through inspection of a large corpus of annotated data. We also propose a new probabilistically motivated error metric for use by the natural language processing and information retrieval communities, intended to supersede precision and recall for appraising segmentation algorithms. Qualitative assessment of our algorithm as well as evaluation using this new metric demonstrate the effectiveness of our approach in two very different domains, Wall Street Journal articles and the TDT Corpus, a collection of newswire articles and broadcast news transcripts.}, author = {Beeferman, Doug and Berger, Adam and Lafferty, John }, citeulike-article-id = {596223}, eprint = {cmp-lg/9706016}, keywords = {algorithms, chunking, segmentation}, month = {Jun}, posted-at = {2008-06-27 16:06:22}, priority = {2}, title = {Text Segmentation Using Exponential Models}, url = {http://arxiv.org/abs/cmp-lg/9706016}, year = {1997} }

TY - ELEC ID - citeulike:596223 L3 - citeulike-article-id:596223 TI - Text Segmentation Using Exponential Models N2 - This paper introduces a new statistical approach to partitioning text automatically into coherent segments. Our approach enlists both short-range and long-range language models to help it sniff out likely sites of topic changes in text. To aid its search, the system consults a set of simple lexical hints it has learned to associate with the presence of boundaries through inspection of a large corpus of annotated data. We also propose a new probabilistically motivated error metric for use by the natural language processing and information retrieval communities, intended to supersede precision and recall for appraising segmentation algorithms. Qualitative assessment of our algorithm as well as evaluation using this new metric demonstrate the effectiveness of our approach in two very different domains, Wall Street Journal articles and the TDT Corpus, a collection of newswire articles and broadcast news transcripts. KW - algorithms KW - chunking KW - segmentation AU - Beeferman, Doug AU - Berger, Adam AU - Lafferty, John PY - 1997/Jun/13/ UR - http://arxiv.org/abs/cmp-lg/9706016 ER -