LAB-3_Extraction of Raw Data
from google.colab import files
from nltk.tokenize import word_tokenize
import nltk
# Step 1: Upload the file
print("Upload your .txt file:")
uploaded = files.upload()
# Step 2: Retrieve the file name
filename = list(uploaded.keys())[0]
print(f"File {filename} uploaded successfully!")
# Step 3: Read the file
print("\nReading the file...")
with open(filename, 'r', encoding='utf-8', errors='ignore') as file:
raw_data = file.read()
print("\nExtracted Raw Data:")
print(raw_data)
# Step 4: Convert Raw Data into Tokens
print("\nConverting Raw Data into Tokens...")
# Download NLTK punkt tokenizer
try:
nltk.download('punkt')
except Exception as e:
print(f"Error downloading punkt tokenizer: {e}")
# Tokenize the raw data
tokens = word_tokenize(raw_data)
print("\nTokens:")
print(tokens)
Additionally, Can compile this code also
import nltk
nltk.download('punkt_tab')
LAB-4_Implementation of Term
Weighting
from sklearn.feature_extraction.text import TfidfVectorizer
# Sample documents
documents = [
"Information retrieval is the process of finding relevant
information.",
"The retrieval process involves techniques like term weighting and
ranking.",
"Term weighting methods like TF-IDF are used in information
retrieval."
]
# Create a TF-IDF Vectorizer
vectorizer = TfidfVectorizer()
# Fit and transform the documents
tfidf_matrix = vectorizer.fit_transform(documents)
# Get the feature names (terms)
terms = vectorizer.get_feature_names_out()
# Convert the matrix to a dense array and print it
tfidf_dense = tfidf_matrix.toarray()
# Display TF-IDF weights
print("TF-IDF Weights:")
for i, doc in enumerate(tfidf_dense):
print(f"\nDocument {i + 1}:")
for term, weight in zip(terms, doc):
if weight > 0:
print(f" {term}: {weight:.3f}")
_________________________________
LAB-5_Implementation of Text
Processing Model
# Import necessary libraries
from google.colab import files
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
import nltk
# Download required NLTK data
nltk.download('punkt')
nltk.download('stopwords')
# Step 1: Upload the .txt file
print("Upload your .txt file:")
uploaded = files.upload()
# Retrieve the file name
filename = list(uploaded.keys())[0]
print(f"\nFile {filename} uploaded successfully!")
# Step 2: Read the file
with open(filename, 'r', encoding='utf-8', errors='ignore') as file:
raw_data = file.read()
print("\nExtracted Raw Data:")
print(raw_data)
# Step 3: Preprocess the text
def preprocess_text(text):
"""
Preprocess the text data:
1. Lowercase the text
2. Tokenize the text
3. Remove stopwords
4. Perform stemming
"""
# Initialize tools
stop_words = set(stopwords.words('english'))
stemmer = PorterStemmer()
# Tokenize the text
tokens = word_tokenize(text)
# Normalize: convert to lowercase
tokens = [token.lower() for token in tokens if token.isalnum()]
# Remove stopwords
filtered_tokens = [word for word in tokens if word not in
stop_words]
# Apply stemming
stemmed_tokens = [stemmer.stem(word) for word in filtered_tokens]
return stemmed_tokens
# Preprocess the extracted raw data
processed_tokens = preprocess_text(raw_data)
print("\nProcessed Tokens:")
print(processed_tokens)
_________________________________
LAB-6: Implementation of Neural
Network Model
# Import necessary libraries
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, Dense, LSTM, Flatten
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from google.colab import files
import numpy as np
# Step 1: Upload a .txt file
print("Upload your .txt file (with labeled data, e.g., text,label):")
uploaded = files.upload()
# Retrieve the file name
filename = list(uploaded.keys())[0]
print(f"\nFile {filename} uploaded successfully!")
# Step 2: Read and process the file
texts, labels = [], []
with open(filename, 'r', encoding='utf-8', errors='ignore') as file:
for line in file:
parts = line.strip().split(",")
if len(parts) == 2: # Ensure there is both text and label
text, label = parts[0].strip(), parts[1].strip()
try:
labels.append(int(label)) # Convert label to integer
texts.append(text)
except ValueError:
print(f"Skipping invalid label: {label}")
else:
print(f"Skipping malformed line: {line.strip()}")
# Validate the data
if len(texts) < 2:
raise ValueError("Not enough data. Ensure the file contains at
least 2 valid text-label pairs.")
# Display sample texts and labels
print("\nSample Texts and Labels:")
print(texts[:5], labels[:5])
# Step 3: Tokenize the text
tokenizer = Tokenizer(num_words=5000, oov_token="<OOV>")
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
# Pad sequences to ensure uniform input size
max_length = 100
padded_sequences = pad_sequences(sequences, maxlen=max_length,
padding='post')
# Convert labels to a NumPy array
labels = np.array(labels)
# Step 4: Split data into training and testing sets
train_size = int(len(texts) * 0.8)
if train_size == 0 or len(texts) - train_size == 0:
raise ValueError("Insufficient data for splitting. Ensure there are
enough samples.")
train_data = padded_sequences[:train_size]
train_labels = labels[:train_size]
test_data = padded_sequences[train_size:]
test_labels = labels[train_size:]
print(f"\nTraining Samples: {len(train_data)}, Testing Samples:
{len(test_data)}")
# Step 5: Build a Neural Network Model
model = Sequential([
Embedding(input_dim=5000, output_dim=64, input_length=max_length),
LSTM(64, return_sequences=True),
Flatten(),
Dense(64, activation='relu'),
Dense(1, activation='sigmoid') # Binary classification
])
model.compile(optimizer='adam', loss='binary_crossentropy',
metrics=['accuracy'])
# Step 6: Train the Model
print("\nTraining the model...")
batch_size = min(len(train_data), 32) # Ensure batch_size is not
larger than the training set
history = model.fit(train_data, train_labels, epochs=5,
validation_data=(test_data, test_labels), batch_size=batch_size)
# Step 7: Evaluate the Model
print("\nEvaluating the model...")
loss, accuracy = model.evaluate(test_data, test_labels)
print(f"Test Loss: {loss}, Test Accuracy: {accuracy}")
# Step 8: Make Predictions
print("\nMaking predictions on test data...")
predictions = (model.predict(test_data) > 0.5).astype("int32")
print("\nSample Predictions:")
for i in range(min(5, len(test_data))):
print(f"Text: {texts[train_size + i]}, Actual Label:
{test_labels[i]}, Predicted Label: {predictions[i][0]}")
.TXT file content:
I love this product,1 This is the worst
experience ever,0 Absolutely fantastic!
Highly recommend.,1 Not worth the
money.,0
—-----------------------------------------------------
Note: Similar to that one, you can
create own text file and compile the
program
################ LAB-7
##########################
# Step 1: Install necessary libraries
!pip install tqdm --quiet # For progress bars
# Step 2: Upload a sample text file
from google.colab import files
print("Upload a sample text file for indexing (e.g., a .txt file).")
uploaded_files = files.upload()
# Read the uploaded file
file_content = None
file_name = list(uploaded_files.keys())[0]
with open(file_name, 'r', encoding='utf-8') as file:
file_content = file.readlines()
# Step 3: Scalable Indexing Implementation
from collections import defaultdict
from tqdm import tqdm
class ScalableIndexer:
def __init__(self): # Corrected constructor
self.index = defaultdict(list) # Dictionary for word-to-line
mapping
def index_file(self, file_content):
"""Indexes a text file line by line."""
for line_num, line in enumerate(tqdm(file_content,
desc="Indexing lines")):
words = line.strip().split()
for word in words:
self.index[word.lower()].append(line_num) # Convert
words to lowercase for case-insensitive indexing
def search(self, term):
"""Searches for a term in the indexed data."""
term = term.lower()
if term in self.index:
return self.index[term]
else:
return []
# Step 4: Index the uploaded file
indexer = ScalableIndexer()
indexer.index_file(file_content)
# Step 5: Perform a search
search_term = input("Enter a word to search for: ")
search_results = indexer.search(search_term)
# Display results
if search_results:
print(f"Found '{search_term}' in lines: {search_results}")
print("\nLines containing the term:")
for line_num in search_results:
print(f"Line {line_num + 1}: {file_content[line_num].strip()}")
else:
print(f"'{search_term}' not found in the file.")
INPUT:
Create own .TXT file and add few lines in TXT file,
then search the term in TXT file
