09-4-文本聚类实战：用K-means对豆瓣读书数据聚类

机器学习与自然语言处理Python自然语言处理第十章课前回顾读取数据分词以及停用词过滤模型训练与预测评估指标文本聚类实战：用K-means对豆瓣读书数据聚类04代码实现流程本章小结实现代码数据源data.csv：实现代码代码组织：-clustering.py：数据读取、聚类以及结果展示-data_loader.py：读取停用词列表-data_processing.py：数据清洗以及数据集转化实现代码DATA_DIR=data_loader.DATA_DIRstopwords=data_loader.get_stopwords(DATA_DIR)book_data=pd.read_csv(DATA_DIR+"/data.csv")book_titles=book_data['title'].tolist()book_content=book_data['content'].tolist()norm_book_content=data_processing.norm_corpus(book_content,stopwords)feature_matrix,_,vectorizer=data_processing.convert_data(norm_book_content,[""],CountVectorizer())

clustering.py数据格式转换实现代码print(book_data.iloc[0],'\n')#查看特征数量print("数据集维度：",feature_matrix.shape)print("部分词特征：\n",vectorizer.get_feature_names()[-10:])

clustering.py打印实现代码defget_cluster_data(clustering_obj,book_data,feature_names,num_clusters,topn_features=10):cluster_details={}ordered_centroids=clustering_obj.cluster_centers_.argsort()[:,::-1]

forcluster_numinrange(num_clusters):cluster_details[cluster_num]={}cluster_details[cluster_num]['cluster_num']=cluster_numkey_features=[feature_names[index]forindexinordered_centroids[cluster_num,:topn_features]]cluster_details[cluster_num]['key_features']=key_featuresbooks=book_data[book_data['Cluster']==cluster_num]['title'].values.tolist()cluster_details[cluster_num]['books']=booksreturncluster_details

clustering.py进行聚类操作实现代码defkmeans_clustering(feature_matrix,num_clusters=10):km=KMeans(n_clusters=num_clusters)#初始化KMeanskm.fit(feature_matrix)#聚类

clusters=km.labels_#获取聚类结果

returnkm,clustersdefprint_cluster_data(cluster_data):#printclusterdetailsforcluster_num,cluster_detailsincluster_data.items():print('Cluster{}details:'.format(cluster_num))print('-'*20)print('Keyfeatures:',cluster_details['key_features'])print('bookinthiscluster:')print("《"+'》,《'.join(cluster_details['books'])+"》")print('='*40)

clustering.py进行聚类操作实现代码num_clusters=10km_obj,clusters=kmeans_clustering(feature_matrix=feature_matrix,num_clusters=num_clusters)book_data['Cluster']=clustersc=Counter(clusters)forcid,countsinsorted(c.items()):print("簇id:{}文档数量:{}".format(cid,counts),end="\n")print("==========\n簇详情：")cluster_data=get_cluster_data(clustering_obj=km_obj,book_data=book_data,feature_names=feature_names,num_clusters=num_clusters,topn_features=5)print_cluster_data(cluster_data)

clustering.py开