
import pandas as pd 
import numpy as np


df = pd.read_csv("./dataset/datas/ml-latest-small/ratings.csv")

df


df.describe()


# 取评分大于均值的数据
df_new = df[df["rating"] >= df["rating"].mean()]


df_new


# 按照用户id聚合，得到userid， movieids
df_group = df_new.groupby("userId")["movieId"].apply(lambda x : " ".join([str(m) for m in x])).reset_index()


df_group


df_group.to_csv("./df_userid_movieids.csv",index=False)


import findspark
findspark.init()


from pyspark.sql import SparkSession
spark = SparkSession \
        .builder \
        .appName("pyspark item2vec") \
        .getOrCreate()


sc = spark.sparkContext


from pyspark.ml.feature import Word2Vec
import  pyspark.sql.functions as F


df_uid_movieids = spark.read.csv("./df_userid_movieids.csv",header=True)


df_uid_movieids.show(5)

+------+--------------------+
|userId|             movieId|
+------+--------------------+
|     1|1 3 6 47 50 101 1...|
|     2|333 1704 3578 687...|
|     3|849 1587 2288 285...|
|     4|106 125 162 176 2...|
|     5|1 21 34 36 50 58 ...|
+------+--------------------+
only showing top 5 rows


df_uid_movieids = df_uid_movieids.withColumn("movie_ids", F.split(df_uid_movieids.movieId," "))


df_uid_movieids.show(5)

+------+--------------------+--------------------+
|userId|             movieId|           movie_ids|
+------+--------------------+--------------------+
|     1|1 3 6 47 50 101 1...|[1, 3, 6, 47, 50,...|
|     2|333 1704 3578 687...|[333, 1704, 3578,...|
|     3|849 1587 2288 285...|[849, 1587, 2288,...|
|     4|106 125 162 176 2...|[106, 125, 162, 1...|
|     5|1 21 34 36 50 58 ...|[1, 21, 34, 36, 5...|
+------+--------------------+--------------------+
only showing top 5 rows


word2v = Word2Vec(
    minCount=0,
    inputCol="movie_ids",
    outputCol="movie_vec"
)


model = word2v.fit(df_uid_movieids)


model.getVectors().show(3,truncate=False)

+-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|word |vector                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|26985|[0.0050064753741025925,-0.002028297632932663,-0.004503972828388214,-5.007848376408219E-4,-0.0018048848723992705,-2.1546287462115288E-4,0.004060663748532534,-2.4738319916650653E-4,0.006108861416578293,0.012412973679602146,-0.005145900882780552,0.008894488215446472,0.009390142746269703,0.00376724055968225,7.11179687641561E-4,0.002031924668699503,-0.0034186281263828278,0.0014267143560573459,-0.004441220313310623,0.0026932843029499054,0.0022411015816032887,0.004398340824991465,0.007277578115463257,0.005493707023561001,0.0020433119498193264,-0.0021653124131262302,-0.005178747233003378,0.0032734284177422523,-4.4577132211998105E-4,0.0027769231237471104,-0.0031353961676359177,0.005101427435874939,0.001588252722285688,-0.009283848106861115,-0.00750300008803606,-0.0027402136474847794,-0.004532665014266968,-0.005247699562460184,-0.0029466692358255386,-0.0016246848972514272,8.531271014362574E-4,-0.0038898088969290257,-0.0057948012836277485,0.002725968835875392,0.004194851033389568,-9.690513834357262E-4,0.007910536602139473,-0.00802365317940712,-0.017484979704022408,-0.0017741742776706815,-0.0030230125412344933,-0.004804839845746756,0.0018083080649375916,-0.0033947303891181946,-0.001476178178563714,0.002190063474699855,0.008550690487027168,-0.009954387322068214,-0.004505809396505356,-0.0030176129657775164,-0.0015940970042720437,0.006996306590735912,0.008167452178895473,0.0032389310654252768,9.914394468069077E-4,0.005043367855250835,-0.0028300927951931953,0.01106351613998413,0.010234171524643898,-5.164177855476737E-5,-0.0062374635599553585,-0.006083593238145113,-1.5201076166704297E-4,0.003443552181124687,6.232280866242945E-4,-4.5204971684142947E-4,0.00267234118655324,0.004803840070962906,0.00798890832811594,-2.1789572201669216E-5,-0.004710946697741747,-0.010331152938306332,0.00829444732517004,-0.008732934482395649,0.004671536386013031,-0.0093796132132411,-6.196596659719944E-4,0.008758122101426125,6.54840434435755E-4,-0.0033783013932406902,8.217996219173074E-5,-0.004704768303781748,-0.00830982904881239,0.0067285653203725815,-0.009113965556025505,-7.263795123435557E-4,-0.0035904315300285816,-0.0020726339425891638,0.007935043424367905,-0.003621063893660903]|
|5451 |[0.0031055069994181395,-0.0035000210627913475,-0.008496161550283432,-0.004185812082141638,-0.008866862393915653,5.353034939616919E-4,0.003553590038791299,7.406119257211685E-4,0.0013319358695298433,0.007023359648883343,-0.002658338285982609,-0.0035089030861854553,0.0042486595921218395,-0.005416847765445709,1.8774339696392417E-4,0.008836913853883743,-0.004842996131628752,-0.0024531648959964514,0.004215316381305456,0.0051176222041249275,0.001406125258654356,0.01072723139077425,0.007557476870715618,0.0019888596143573523,0.004980228375643492,-0.008225614205002785,-0.0010050476994365454,0.00966300442814827,0.0077622756361961365,0.004254800267517567,-0.003096851287409663,4.700117278844118E-4,0.006533435545861721,-0.007814507931470871,-0.007216757163405418,0.00917189009487629,-0.007343550678342581,-0.004643237683922052,-0.00533076748251915,0.005876368377357721,-0.002618091879412532,-0.006996131502091885,-0.003387381322681904,0.006856309715658426,0.0017869183793663979,0.0034254840575158596,-0.00239483080804348,-0.002244992647320032,-0.012762807309627533,-0.0025836187414824963,0.008910615928471088,-0.006975111085921526,0.003355941269546747,-0.0028422020841389894,-0.001240801066160202,-0.002100682118907571,-1.8520187586545944E-4,-9.587303502485156E-4,0.0012905766488984227,-0.002805916592478752,0.0026619615964591503,0.003241271246224642,-0.0027410194743424654,0.005146475043147802,-4.2761676013469696E-5,-0.002588769420981407,0.005873008165508509,0.004935290664434433,0.007206003647297621,0.007419294212013483,-8.414408657699823E-4,0.004214360378682613,3.7713436177000403E-4,0.006916271522641182,-0.003907330334186554,-0.001082056201994419,-9.230724535882473E-4,0.007022482343018055,0.006507051642984152,-0.00805045384913683,-0.006945343222469091,-0.0011496240040287375,0.0019969316199421883,-0.0016379388980567455,0.0026020195800811052,-4.79996670037508E-5,0.003442800836637616,0.0074386498890817165,-0.0023196127731353045,0.003550850786268711,-0.00415504164993763,-0.005467056296765804,-9.970786049962044E-4,-9.014364331960678E-4,-0.0038271595258265734,4.1530514135956764E-4,-0.008489049971103668,-0.0036829945165663958,0.006188772153109312,6.98166317306459E-4]     |
|4018 |[0.002354368567466736,-0.031213808804750443,-0.018744662404060364,-0.012790479697287083,-0.030172502622008324,-5.249088280834258E-4,0.02637300454080105,0.017274275422096252,0.0151343559846282,-0.008900091052055359,-0.025969771668314934,-0.03866435959935188,0.04447852820158005,-0.02605719305574894,-0.009259299375116825,0.04372621327638626,-0.011331148445606232,-0.03336198255419731,-0.003730029333382845,0.013608603738248348,0.03701106831431389,0.056183911859989166,0.014508765190839767,0.023705516010522842,0.016241252422332764,-0.02544344775378704,-0.04515290632843971,0.028659144416451454,0.03223863244056702,0.012135575525462627,-0.016723010689020157,-0.002531433943659067,0.04097439721226692,-0.04418302699923515,-0.019686061888933182,0.056181054562330246,-0.043848879635334015,-0.030469784513115883,-0.013955106027424335,0.019655484706163406,-0.005234271753579378,-0.033944934606552124,-0.03353076055645943,0.010481216944754124,0.04236995428800583,0.009512104094028473,0.0012433665106073022,-0.02115059271454811,-0.05864269286394119,-0.02713056653738022,0.024782834574580193,-0.036645155400037766,0.034183572977781296,-0.03304430469870567,0.018561052158474922,-0.0010801500175148249,-0.017815429717302322,0.005534081719815731,0.005291842855513096,-0.031397584825754166,-5.074883229099214E-4,0.0110480273142457,-0.0067872474901378155,0.0031418143771588802,0.004469644743949175,-0.00676601380109787,0.025669768452644348,0.033723026514053345,0.035603366792201996,0.00904915016144514,-0.029603654518723488,0.018855009227991104,0.0040616365149617195,0.0311270821839571,-0.04569743573665619,-0.027031932026147842,0.012527135200798512,0.020414073020219803,0.00464828684926033,-0.030177727341651917,-0.018233517184853554,-0.002253240440040827,0.013126358389854431,-0.0146793182939291,-0.005717835389077663,-0.01065471675246954,0.0041510737501084805,0.009938178583979607,0.0022606137208640575,0.009388011880218983,-0.05765107274055481,-0.01155717484652996,0.02764756977558136,-0.005231184884905815,-0.04534970223903656,0.016748875379562378,-0.01907907798886299,0.007882136851549149,-0.0034679672680795193,0.02240237034857273]                                                       |
+-----+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
only showing top 3 rows


df_movie_embedding = model.getVectors().select("word","vector") \
        .toPandas()


df_movie_embedding


df_movie = pd.read_csv("./dataset/datas/ml-latest-small/movies.csv")


df_movie


df_movie.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 9742 entries, 0 to 9741
Data columns (total 3 columns):
 #   Column   Non-Null Count  Dtype 
---  ------   --------------  ----- 
 0   movieId  9742 non-null   int64 
 1   title    9742 non-null   object
 2   genres   9742 non-null   object
dtypes: int64(1), object(2)
memory usage: 228.5+ KB


df_movie["movieId"] =df_movie["movieId"].astype(str)


df_merge = pd.merge(left=df_movie_embedding,right=df_movie, left_on="word",right_on="movieId")


df_merge.columns

Index(['word', 'vector', 'movieId', 'title', 'genres'], dtype='object')


df_merge = df_merge[['movieId', 'title', 'genres', 'vector']]


df_merge


# 选定一个电影 104419
movie_id= "104419"


df_merge[df_merge["movieId"]== movie_id]


df_merge[df_merge["movieId"]== movie_id]["vector"].iloc[0]

DenseVector([0.0029, -0.004, -0.0045, 0.0018, 0.0004, 0.0007, 0.0027, 0.0069, 0.0033, 0.0039, -0.0094, -0.0004, 0.0106, -0.002, -0.0011, 0.0074, -0.0094, -0.0005, 0.002, -0.0056, 0.0045, 0.0087, 0.0099, 0.001, -0.0011, -0.0, -0.0035, 0.0114, 0.0075, 0.0032, -0.001, 0.0033, 0.0035, -0.0094, -0.0011, 0.0053, -0.0089, -0.0013, -0.0056, -0.0036, 0.0017, -0.0101, -0.0127, 0.0005, 0.0106, 0.0046, 0.0073, -0.0122, -0.0207, -0.0026, 0.0057, -0.0074, 0.0017, -0.0131, 0.0046, 0.0088, 0.0062, 0.0011, 0.005, -0.0052, -0.0028, 0.0039, 0.0023, -0.0043, 0.0001, 0.0022, 0.0021, 0.0111, 0.0102, 0.0049, -0.0073, 0.0054, -0.003, 0.0112, -0.0003, -0.0005, -0.0015, 0.0038, 0.0104, -0.0027, -0.0114, -0.0089, 0.0079, -0.0095, 0.004, -0.0077, -0.0006, 0.0098, 0.0005, 0.0048, -0.0059, -0.0049, 0.001, 0.002, -0.0135, 0.0035, -0.0035, -0.0016, -0.0022, 0.0041])


target_movie_emb= df_merge[df_merge["movieId"]== movie_id]["vector"].iloc[0]


# 余弦相似度， 与target——movieid 104419的相似度
from scipy.spatial import distance
df_merge["sim_value"] = df_merge["vector"].map(lambda x : 1- distance.cosine(target_movie_emb,x))


df_merge


# 按相似度降序排列，查询前10条
df_merge.sort_values("sim_value",ascending=False)[['movieId', 'title', 'genres','sim_value']].head(10)

	userId	movieId	rating	timestamp
0	1	1	4.0	964982703
1	1	3	4.0	964981247
2	1	6	4.0	964982224
3	1	47	5.0	964983815
4	1	50	5.0	964982931
...	...	...	...	...
100831	610	166534	4.0	1493848402
100832	610	168248	5.0	1493850091
100833	610	168250	5.0	1494273047
100834	610	168252	5.0	1493846352
100835	610	170875	3.0	1493846415

	userId	movieId	rating	timestamp
count	100836.000000	100836.000000	100836.000000	1.008360e+05
mean	326.127564	19435.295718	3.501557	1.205946e+09
std	182.618491	35530.987199	1.042529	2.162610e+08
min	1.000000	1.000000	0.500000	8.281246e+08
25%	177.000000	1199.000000	3.000000	1.019124e+09
50%	325.000000	2991.000000	3.500000	1.186087e+09
75%	477.000000	8122.000000	4.000000	1.435994e+09
max	610.000000	193609.000000	5.000000	1.537799e+09

	userId	movieId	rating	timestamp
0	1	1	4.0	964982703
1	1	3	4.0	964981247
2	1	6	4.0	964982224
3	1	47	5.0	964983815
4	1	50	5.0	964982931
...	...	...	...	...
100830	610	166528	4.0	1493879365
100831	610	166534	4.0	1493848402
100832	610	168248	5.0	1493850091
100833	610	168250	5.0	1494273047
100834	610	168252	5.0	1493846352

	word	vector
0	26985	[0.0050064753741025925, -0.002028297632932663,...
1	5451	[0.0031055069994181395, -0.0035000210627913475...
2	4018	[0.002354368567466736, -0.031213808804750443, ...
3	4056	[0.0064166150987148285, -0.025528382509946823,...
4	32584	[-0.002056463621556759, 0.002898153616115451, ...
...	...	...
6293	104419	[0.0028946588281542063, -0.004049575887620449,...
6294	133867	[0.0029085518326610327, -0.007749718148261309,...
6295	3439	[0.0034692941699177027, -0.0001765647029969841...
6296	2141	[0.01832941360771656, -0.018213754519820213, -...
6297	111364	[0.003449969459325075, 0.0009291048627346754, ...

	movieId	title	genres
0	1	Toy Story (1995)	Adventure\|Animation\|Children\|Comedy\|Fantasy
1	2	Jumanji (1995)	Adventure\|Children\|Fantasy
2	3	Grumpier Old Men (1995)	Comedy\|Romance
3	4	Waiting to Exhale (1995)	Comedy\|Drama\|Romance
4	5	Father of the Bride Part II (1995)	Comedy
...	...	...	...
9737	193581	Black Butler: Book of the Atlantic (2017)	Action\|Animation\|Comedy\|Fantasy
9738	193583	No Game No Life: Zero (2017)	Animation\|Comedy\|Fantasy
9739	193585	Flint (2017)	Drama
9740	193587	Bungo Stray Dogs: Dead Apple (2018)	Action\|Animation
9741	193609	Andrew Dice Clay: Dice Rules (1991)	Comedy

知识：¶

数据整理¶

pyspark.ml word2vec¶

对于给定电影算出最相似的10个电影¶

	userId	movieId
0	1	1 3 6 47 50 101 110 151 157 163 216 231 235 26...
1	2	333 1704 3578 6874 46970 48516 58559 60756 681...
2	3	849 1587 2288 2851 3024 3703 4518 5181 5746 57...
3	4	106 125 162 176 215 232 260 265 319 342 345 34...
4	5	1 21 34 36 50 58 110 232 247 261 290 296 367 4...
...	...	...
604	606	17 18 29 32 46 50 68 70 73 80 82 147 154 156 1...
605	607	1 36 86 110 150 165 188 241 292 318 366 377 38...
606	608	10 16 47 50 110 170 172 293 296 318 333 353 38...
607	609	10 253 296 318 356 457 590 731 1150 1161
608	610	1 6 16 32 47 50 70 110 111 112 194 215 231 260...

	movieId	title	genres	vector
0	26985	Nirvana (1997)	Action\|Sci-Fi	[0.0050064753741025925, -0.002028297632932663,...
1	5451	Pumpkin (2002)	Comedy\|Drama\|Romance	[0.0031055069994181395, -0.0035000210627913475...
2	4018	What Women Want (2000)	Comedy\|Romance	[0.002354368567466736, -0.031213808804750443, ...
3	4056	Pledge, The (2001)	Crime\|Drama\|Mystery\|Thriller	[0.0064166150987148285, -0.025528382509946823,...
4	32584	Ballad of Jack and Rose, The (2005)	Drama	[-0.002056463621556759, 0.002898153616115451, ...
...	...	...	...	...
6293	104419	Justice League: Crisis on Two Earths (2010)	Action\|Animation\|Sci-Fi	[0.0028946588281542063, -0.004049575887620449,...
6294	133867	Barely Lethal (2015)	Action\|Adventure\|Comedy	[0.0029085518326610327, -0.007749718148261309,...
6295	3439	Teenage Mutant Ninja Turtles II: The Secret of...	Action\|Children\|Fantasy	[0.0034692941699177027, -0.0001765647029969841...
6296	2141	American Tail, An (1986)	Adventure\|Animation\|Children\|Comedy	[0.01832941360771656, -0.018213754519820213, -...
6297	111364	Godzilla (2014)	Action\|Adventure\|Sci-Fi\|IMAX	[0.003449969459325075, 0.0009291048627346754, ...

	movieId	title	genres	sim_value
6293	104419	Justice League: Crisis on Two Earths (2010)	Action\|Animation\|Sci-Fi	1.000000
5423	47629	The Queen (2006)	Drama	0.898540
4348	43921	Running Scared (2006)	Action\|Crime\|Thriller	0.895997
4589	79592	Other Guys, The (2010)	Action\|Comedy	0.894612
3198	37733	History of Violence, A (2005)	Action\|Crime\|Drama\|Thriller	0.893884
3191	78469	A-Team, The (2010)	Action\|Comedy\|Thriller	0.890925
5013	37729	Corpse Bride (2005)	Animation\|Comedy\|Fantasy\|Musical\|Romance	0.890434
6083	7451	Mean Girls (2004)	Comedy	0.889519
6159	7247	Chitty Chitty Bang Bang (1968)	Adventure\|Children\|Comedy\|Fantasy\|Musical	0.885103
4400	26578	Sacrifice, The (Offret - Sacraficatio) (1986)	Drama	0.884508