datalife2019.wiki
1 %title My Life in Data 2019 2 :blog:dev:python: 3 %date 2020-03-03 20:48 4 %update 2023-06-03 22:31 5 6 This was originally written in [Jupyter](https://jupyter.org/) in a sort of literate programming style. 7 8 ---- 9 10 Originally I intended this post to go up in January as a year in review (ideally it would've been completed by then) but now it's March so like fuckit I'm just posting what I got through. 11 12 I have been collecting data about my life for some time now, and I thought it would be cool to see what it says about me. Here is the Jupyter Notebook I used to gain some insight into my 2019. These numbers probably won't make much sense to you until I get to explaining the process and what they mean. 13 14 = Common functions = 15 16 {{{python 17 import json 18 from tasklib import TaskWarrior 19 import datetime 20 import pandas as pd 21 import matplotlib.pyplot as plt 22 import re 23 from sklearn.feature_extraction.text import CountVectorizer 24 from sklearn.feature_extraction.text import TfidfVectorizer 25 }}} 26 27 28 {{{python 29 def only2019(df): 30 return df.filter(like='2019', axis=0) 31 }}} 32 33 = Journal = 34 35 == import journal into habits data == 36 37 == TF-IDF for jrnl == 38 39 https://kavita-ganesan.com/extracting-keywords-from-text-tfidf/ 40 41 42 {{{python 43 def pre_process(text): 44 # lowercase 45 text=text.lower() 46 #remove tags 47 #text=re.sub("<!--?.*?-->","",text) 48 # remove special characters and digits 49 text=re.sub("(\\d|\\W)+"," ",text) 50 return text 51 52 def sort_coo(coo_matrix): 53 tuples = zip(coo_matrix.col, coo_matrix.data) 54 return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True) 55 56 def extract_topn_from_vector(feature_names, sorted_items, topn=10): 57 """get the feature names and tf-idf score of top n items""" 58 59 #use only topn items from vector 60 sorted_items = sorted_items[:topn] 61 62 score_vals = [] 63 feature_vals = [] 64 65 # word index and corresponding tf-idf score 66 for idx, score in sorted_items: 67 68 #keep track of feature name and its corresponding score 69 score_vals.append(round(score, 3)) 70 feature_vals.append(feature_names[idx]) 71 72 #create a tuples of feature,score 73 #results = zip(feature_vals,score_vals) 74 results= {} 75 for idx in range(len(feature_vals)): 76 results[feature_vals[idx]]=score_vals[idx] 77 78 return results 79 80 }}} 81 82 83 {{{python 84 with open('data/tiddlers.json' , 'r') as file: 85 tiddly = json.load(file) 86 87 jrnl = pd.DataFrame.from_dict(tiddly) 88 jrnl.set_axis(jrnl['title'], axis='index', inplace=True) 89 jrnl = only2019(jrnl.drop(['created', 'modified'], axis=1)) 90 jrnl.drop(['title', 'tags'], axis=1, inplace=True) 91 jrnl['text'] = jrnl['text'].apply(lambda x: pre_process(x)) 92 93 #get the text column 94 docs=jrnl['text'].tolist() 95 96 tfidf = TfidfVectorizer(stop_words='english') 97 X = tfidf.fit_transform(docs) 98 99 #sort the tf-idf vectors by descending order of scores 100 sorted_items=sort_coo(X.tocoo()) 101 102 #extract only the top n; n here is 10 103 keywords=extract_topn_from_vector(tfidf.get_feature_names(),sorted_items,40) 104 105 # now print the results 106 #print("\n=====Doc=====") 107 #print(doc) 108 print("\n===Keywords===") 109 for k in keywords: 110 print(k,keywords[k]) 111 }}} 112 113 114 ===Keywords=== 115 rain 0.699 116 years 0.658 117 spokes 0.584 118 new 0.567 119 dont 0.545 120 quite 0.507 121 feeling 0.507 122 freedom 0.502 123 despair 0.502 124 awaits 0.502 125 voices 0.501 126 hearing 0.501 127 hard 0.495 128 better 0.494 129 news 0.488 130 uber 0.469 131 pay 0.468 132 graveyard 0.443 133 cult 0.443 134 repeatedly 0.437 135 tech 0.433 136 laurel 0.431 137 wedding 0.426 138 wont 0.422 139 bit 0.418 140 tour 0.404 141 able 0.394 142 strategy 0.392 143 like 0.392 144 clouds 0.383 145 travelling 0.383 146 watching 0.383 147 sources 0.377 148 flows 0.377 149 doodling 0.374 150 successful 0.373 151 imagine 0.373 152 ye 0.373 153 tell 0.371 154 im 0.37 155 156 157 According to habits I have 49 Journal entries 158 159 160 Let's compare that with this data 161 162 163 {{{python 164 import json 165 }}} 166 167 168 {{{python 169 with open('data/tiddlers.json' , 'r') as file: 170 tiddly = json.load(file) 171 172 jrnl = pd.DataFrame.from_dict(tiddly) 173 jrnl.set_axis(jrnl['title'], axis='index', inplace=True) 174 jrnl = only2019(jrnl.drop(['created', 'modified'], axis=1)) 175 jrnl.count() 176 }}} 177 178 179 180 181 tags 87 182 text 88 183 title 88 184 dtype: int64 185 186 187 188 189 {{{python 190 sameDay = jrnl 191 sameDay['title'] = sameDay.title.map(lambda x: x[:-8]) 192 sameDay[sameDay['title'].duplicated(keep=False)].count() 193 }}} 194 195 196 197 198 tags 30 199 text 30 200 title 30 201 dtype: int64 202 203 204 205 Where are the [87-15=(73)] - 49 = 23 entries that exist but aren't in my loop Habits? 206 207 23 Entries before March? 208 209 210 {{{python 211 def beforeMarch(df): 212 return df.filter(regex='(January|Febuary|March)', axis=0) 213 }}} 214 215 216 {{{python 217 beforeMarch(jrnl).count() 218 }}} 219 220 221 222 223 tags 23 224 text 23 225 title 23 226 dtype: int64 227 228 229 230 # Taskwarrior 231 232 233 {{{python 234 import json 235 from tasklib import TaskWarrior 236 import datetime 237 import pandas as pd 238 import matplotlib.pyplot as plt 239 }}} 240 241 Month Added Completed Deleted Net 242 --------- ----- --------- ------- --- 243 January 18 19 0 -1 244 February 9 7 0 2 245 March 27 23 0 4 246 April 13 14 3 -4 247 May 10 9 0 1 248 June 17 13 0 4 249 July 17 17 0 0 250 August 35 37 0 -2 251 September 22 24 1 -3 252 October 19 15 2 2 253 November 17 16 0 1 254 December 37 42 1 -6 255 256 Total 241 236 7 5 257 258 {{{python 259 with open('data/task.json', 'r') as myfile: 260 task = json.load(myfile) 261 262 # Convert the data into a data frame 263 # Some preliminary analysis 264 tasks = pd.DataFrame.from_dict(task) 265 tasks.set_axis(tasks['entry'], axis='index', inplace=True) 266 tasks = tasks.drop(['annotations', 'depends', 'parent', 'uuid', 'entry'], axis=1) 267 only2019(tasks).count() 268 }}} 269 270 271 272 273 description 241 274 due 107 275 end 241 276 id 241 277 imask 0 278 mask 0 279 modified 241 280 priority 3 281 project 188 282 recur 0 283 status 241 284 tags 215 285 until 0 286 urgency 241 287 dtype: int64 288 289 290 291 292 {{{python 293 # How many of each tag did I do? 294 295 l = only2019(tasks).tags.dropna().to_list() 296 flat_list = [item for sublist in l for item in sublist] 297 print(str(len(set(flat_list))) + " unique tags") 298 print(str(len(l)) + " tagged items") 299 print("The tags are: " + str(set(flat_list))) 300 301 fig = plt.figure(figsize=(14,8)) 302 plt.hist(flat_list, rwidth=1/3, align='left', bins=16) 303 plt.show() 304 }}} 305 306 16 unique tags 307 215 tagged items 308 The tags are: {'rocks', 'friends', 'd.tech', 'chores', 'job', 'prpj', 'fam', 'artifex', 'contact', 'd.infra', 'fun', 'work', 'uni', 'life', 'travel'} 309 310 311 312  313 314 315 316 {{{python 317 # Description mining 318 319 l = only2019(tasks).description 320 print(l.describe()) 321 print() 322 print("Repeated descriptions: ") 323 for e in l[l.duplicated(keep=False)].unique(): 324 print(" " + e) 325 }}} 326 327 count 241 328 unique 216 329 top fold clothes 330 freq 7 331 Name: description, dtype: object 332 333 Repeated descriptions: 334 laundry 335 haircut 336 fold clothes 337 recharge my way 338 change sheets 339 cut hair 340 buy condoms 341 schedule counsellor meeting 342 cut nails 343 recharge myway 344 book counsellor meeting 345 shave 346 do laundry 347 348 349 350 {{{python 351 # Most described task? 352 ## TODO 353 }}} 354 355 # Habits 356 357 358 {{{python 359 import pandas as pd 360 import calendar 361 import matplotlib 362 import matplotlib.pyplot as plt 363 }}} 364 365 366 {{{python 367 #colNames = ['date', 'Godmode', 'Meditate', 'Exercise', 'Piano','Read', 'Journal', 'Gratitude', 'devlog','Plants', 'Job', 'Draw'] 368 checks = pd.read_csv('data/LoopHabits/Checkmarks.csv', header=0) 369 checks.set_axis(checks['date'], axis='index', inplace=True) 370 checks = checks.drop(['Godmode', 'Piano', 'Gratitude', 'Job', 'date'], axis=1) 371 df = MarchOnwards(only2019(checks)) 372 print("Days done: " + str(countValue(2, df))) 373 print("Non-streak days: " + str(countZeros(df))) 374 print("Streak days: " + str(countValue(1, df))) 375 print("Longest streak: " + str(df.apply(longestStreak, axis=0))) 376 print("Longest zeros: " + str(df.apply(longestZero, axis=0))) 377 #print("Best month: " + str(sumMonth(df))) 378 sumMonth(df) 379 meanMonth(df) 380 #print("Worst month: " + calendar.month_name[int(sumMonth(df).idxmin())]) 381 #plotScore(df) 382 }}} 383 384 Days done: Meditate 64 385 Exercise 65 386 Read 69 387 Journal 49 388 devlog 32 389 Plants 38 390 Draw 12 391 dtype: int64 392 Non-streak days: Meditate 176 393 Exercise 179 394 Read 161 395 Journal 70 396 devlog 154 397 Plants 120 398 Draw 252 399 dtype: int64 400 Streak days: Meditate 35 401 Exercise 31 402 Read 45 403 Journal 156 404 devlog 89 405 Plants 117 406 Draw 11 407 dtype: int64 408 Longest streak: Meditate 17 409 Exercise 11 410 Read 12 411 Journal 55 412 devlog 29 413 Plants 87 414 Draw 17 415 dtype: int64 416 Longest zeros: Meditate 23 417 Exercise 26 418 Read 43 419 Journal 27 420 devlog 121 421 Plants 54 422 Draw 229 423 dtype: int64 424 425 426 427 428 429 <div> 430 <style scoped> 431 .dataframe tbody tr th:only-of-type { 432 vertical-align: middle; 433 } 434 435 .dataframe tbody tr th { 436 vertical-align: top; 437 } 438 439 .dataframe thead th { 440 text-align: right; 441 } 442 </style> 443 <table border="1" class="dataframe"> 444 <thead> 445 <tr style="text-align: right;"> 446 <th></th> 447 <th>Meditate</th> 448 <th>Exercise</th> 449 <th>Read</th> 450 <th>Journal</th> 451 <th>devlog</th> 452 <th>Plants</th> 453 <th>Draw</th> 454 </tr> 455 <tr> 456 <th>month</th> 457 <th></th> 458 <th></th> 459 <th></th> 460 <th></th> 461 <th></th> 462 <th></th> 463 <th></th> 464 </tr> 465 </thead> 466 <tbody> 467 <tr> 468 <th>4</th> 469 <td>0.133333</td> 470 <td>0.266667</td> 471 <td>0.000000</td> 472 <td>0.900000</td> 473 <td>0.000000</td> 474 <td>1.133333</td> 475 <td>0.000000</td> 476 </tr> 477 <tr> 478 <th>5</th> 479 <td>0.387097</td> 480 <td>0.193548</td> 481 <td>0.193548</td> 482 <td>0.645161</td> 483 <td>0.000000</td> 484 <td>0.516129</td> 485 <td>0.000000</td> 486 </tr> 487 <tr> 488 <th>6</th> 489 <td>0.500000</td> 490 <td>0.266667</td> 491 <td>0.166667</td> 492 <td>0.666667</td> 493 <td>0.000000</td> 494 <td>0.000000</td> 495 <td>0.000000</td> 496 </tr> 497 <tr> 498 <th>7</th> 499 <td>0.516129</td> 500 <td>0.838710</td> 501 <td>0.967742</td> 502 <td>0.774194</td> 503 <td>0.064516</td> 504 <td>0.612903</td> 505 <td>0.000000</td> 506 </tr> 507 <tr> 508 <th>8</th> 509 <td>0.806452</td> 510 <td>0.741935</td> 511 <td>0.935484</td> 512 <td>1.161290</td> 513 <td>1.225806</td> 514 <td>1.193548</td> 515 <td>0.000000</td> 516 </tr> 517 <tr> 518 <th>9</th> 519 <td>1.133333</td> 520 <td>0.600000</td> 521 <td>1.233333</td> 522 <td>1.300000</td> 523 <td>1.066667</td> 524 <td>1.333333</td> 525 <td>0.000000</td> 526 </tr> 527 <tr> 528 <th>10</th> 529 <td>0.774194</td> 530 <td>1.064516</td> 531 <td>0.580645</td> 532 <td>0.935484</td> 533 <td>1.032258</td> 534 <td>1.193548</td> 535 <td>0.000000</td> 536 </tr> 537 <tr> 538 <th>11</th> 539 <td>0.700000</td> 540 <td>0.733333</td> 541 <td>0.800000</td> 542 <td>1.233333</td> 543 <td>1.000000</td> 544 <td>0.333333</td> 545 <td>0.533333</td> 546 </tr> 547 <tr> 548 <th>12</th> 549 <td>0.387097</td> 550 <td>0.548387</td> 551 <td>1.096774</td> 552 <td>0.709677</td> 553 <td>0.612903</td> 554 <td>0.000000</td> 555 <td>0.612903</td> 556 </tr> 557 </tbody> 558 </table> 559 </div> 560 561 562 563 564 {{{python 565 def MarchOnwards(df): 566 # For habits only, lost my phone in late Febuary, didn't have a recent backup 567 return df.filter(regex='[0-9]{4}-(0?[4-9]|1?[0-2])-[0-9]{2}', axis=0) 568 569 def longestZero(df): 570 # reverse the series so dates are ascending (increasing?) 571 # pad out the series with 0s, then diff it to track total, use that total to calculate highest streak 572 # streak increases when sum is 0 573 diffStreak = pd.concat([pd.Series([0]) , df[::-1], pd.Series([0])]).diff().tolist() 574 runningSum = 0 575 streak = 0 576 streakList = [] 577 for e in diffStreak[1:-1]: 578 runningSum += e 579 if runningSum == 0: 580 streak += 1 581 else: 582 streakList.append(streak) 583 streak = 0 584 return max(streakList) 585 586 def longestStreak(df): 587 # reverse the series so dates are ascending (increasing?) 588 # pad out the series with 0s, then diff it to track total, use that total to calculate highest streak 589 # streak increases when sum is non-zero 590 diffStreak = pd.concat([pd.Series([0]) , df[::-1], pd.Series([0])]).diff().tolist() 591 runningSum = 0 592 streak = 0 593 streakList = [] 594 for e in diffStreak[1:-1]: 595 runningSum += e 596 if runningSum != 0: 597 streak += 1 598 else: 599 streakList.append(streak) 600 streak = 0 601 return max(streakList) 602 603 def sumMonth(df): 604 df2 = df.reset_index() 605 df['month'] = pd.DatetimeIndex(df2['date']).month 606 return df.groupby(['month']).sum() 607 608 def meanMonth(df): 609 df2 = df.reset_index() 610 df['month'] = pd.DatetimeIndex(df2['date']).month 611 return df.groupby(['month']).mean() 612 613 def plotScore(df): 614 df = meanMonth(df) 615 plt.plot(df) 616 plt.show() 617 618 def countZeros(df): 619 return countValue(0, df) 620 621 def countValue(countValue, df): 622 return (df == countValue).astype(int).sum() 623 624 }}} 625 626 627 {{{python 628 # checking for CSV of simply one habit 629 630 colNames = ['date', 'value'] 631 meditateChecks = pd.read_csv('LoopHabits/002 Meditate/Checkmarks.csv', names=colNames, header=None) 632 meditateScore = pd.read_csv('LoopHabits/002 Meditate/Scores.csv', names=colNames, header=None) 633 }}} 634 635 636 {{{python 637 # might be broken now 638 # need to deal with dates in columns 639 meditateChecks.set_axis(checks['date'], axis='index', inplace=True) 640 df = MarchOnwards(only2019(meditateChecks)) 641 print("Days done: " + str(countValue(2, df))) 642 print("Non-streak days: " + str(countZeros(df))) 643 print("Streak days: " + str(countValue(1, df))) 644 print("Longest streak: " + str(longestStreak(df))) 645 print("Longest zeros: " + str(longestZero(df))) 646 print("Best month: " + calendar.month_name[int(sumMonth(df).idxmax())]) 647 print("Worst month: " + calendar.month_name[int(sumMonth(df).idxmin())]) 648 plotScore(df) 649 }}} 650 651 652 {{{python 653 # Total days for habits: 306 654 # How come when counting streaks I only get 299? 655 # missing 1 week?? 656 print(72 + 195 + 39) 657 longestStreak(onlyMarch(only2019(meditateChecks))) + longestZero(onlyMarch(only2019(meditateChecks))) 658 }}} 659 660 306 661 662 663 664 665 666 299 667 668 669 670 Why don't the above two numbers match?? 671 672 673 {{{python 674 675 }}} 676 677 ----- 678 = Backlinks = 679 680 - [[241217-1900|Free of the Task List]]