Ref: n-grams in python, four, five, six grams?
Ref: "Elegant n-gram generation in Python"
import nltk
sentence = """At eight o'clock on Thursday morning
Arthur didn't feel very good."""
# 1 gram
tokens = nltk.word_tokenize(sentence)
print("1 gram:\n", tokens, "\n")
# 2 grams
n = 2
tokens_2 = nltk.ngrams(tokens, n)
print("2 grams:\n", [i for i in tokens_2], "\n")
# 3 grams
n = 3
tokens_3 = nltk.ngrams(tokens, n)
print("3 grams:\n", [i for i in tokens_3], "\n")
# 4 grams
n = 4
tokens_4 = nltk.ngrams(tokens, n)
print("4 grams:\n", [i for i in tokens_4], "\n")
outputs:
1 gram:
['At', 'eight', "o'clock", 'on', 'Thursday', 'morning', 'Arthur', 'did', "n't", 'feel', 'very', 'good', '.']
2 grams:
[('At', 'eight'), ('eight', "o'clock"), ("o'clock", 'on'), ('on', 'Thursday'), ('Thursday', 'morning'), ('morning', 'Arthur'), ('Arthur', 'did'), ('did', "n't"), ("n't", 'feel'), ('feel', 'very'), ('very', 'good'), ('good', '.')]
3 grams:
[('At', 'eight', "o'clock"), ('eight', "o'clock", 'on'), ("o'clock", 'on', 'Thursday'), ('on', 'Thursday', 'morning'), ('Thursday', 'morning', 'Arthur'), ('morning', 'Arthur', 'did'), ('Arthur', 'did', "n't"), ('did', "n't", 'feel'), ("n't", 'feel', 'very'), ('feel', 'very', 'good'), ('very', 'good', '.')]
4 grams:
[('At', 'eight', "o'clock", 'on'), ('eight', "o'clock", 'on', 'Thursday'), ("o'clock", 'on', 'Thursday', 'morning'), ('on', 'Thursday', 'morning', 'Arthur'), ('Thursday', 'morning', 'Arthur', 'did'), ('morning', 'Arthur', 'did', "n't"), ('Arthur', 'did', "n't", 'feel'), ('did', "n't", 'feel', 'very'), ("n't", 'feel', 'very', 'good'), ('feel', 'very', 'good', '.')]
Another method to output:
import nltk
sentence = """At eight o'clock on Thursday morning
Arthur didn't feel very good."""
# 1 gram
tokens = nltk.word_tokenize(sentence)
print("1 gram:\n", tokens, "\n")
# 2 grams
n = 2
tokens_2 = nltk.ngrams(tokens, n)
print("2 grams:\n", [' '.join(list(i)) for i in tokens_2], "\n")
# 3 grams
n = 3
tokens_3 = nltk.ngrams(tokens, n)
print("3 grams:\n", [' '.join(list(i)) for i in tokens_3], "\n")
# 4 grams
n = 4
tokens_4 = nltk.ngrams(tokens, n)
print("4 grams:\n", [' '.join(list(i)) for i in tokens_4], "\n")
outputs:
1 gram:
['At', 'eight', "o'clock", 'on', 'Thursday', 'morning', 'Arthur', 'did', "n't", 'feel', 'very', 'good', '.']
2 grams:
['At eight', "eight o'clock", "o'clock on", 'on Thursday', 'Thursday morning', 'morning Arthur', 'Arthur did', "did n't", "n't feel", 'feel very', 'very good', 'good .']
3 grams:
["At eight o'clock", "eight o'clock on", "o'clock on Thursday", 'on Thursday morning', 'Thursday morning Arthur', 'morning Arthur did', "Arthur did n't", "did n't feel", "n't feel very", 'feel very good', 'very good .']
4 grams:
["At eight o'clock on", "eight o'clock on Thursday", "o'clock on Thursday morning", 'on Thursday morning Arthur', 'Thursday morning Arthur did', "morning Arthur did n't", "Arthur did n't feel", "did n't feel very", "n't feel very good", 'feel very good .']