\n",
@@ -5969,9 +5943,17 @@
"[20 rows x 7 columns]"
]
},
+ "execution_count": 119,
"metadata": {},
- "execution_count": 119
+ "output_type": "execute_result"
}
+ ],
+ "source": [
+ "# Create prediction-enriched test dataframe\n",
+ "test_df[\"prediction\"] = test_pred_classes # create column with test prediction class names\n",
+ "test_df[\"pred_prob\"] = tf.reduce_max(test_pred_probs, axis=1).numpy() # get the maximum prediction probability\n",
+ "test_df[\"correct\"] = test_df[\"prediction\"] == test_df[\"target\"] # create binary column for whether the prediction is right or not\n",
+ "test_df.head(20)"
]
},
{
@@ -5987,23 +5969,17 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
- "id": "tDUOsuKJ6IQH",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 417
},
+ "id": "tDUOsuKJ6IQH",
"outputId": "8117b505-1d7b-49dc-9291-a76f92e00e6e"
},
- "source": [
- "# Find top 100 most wrong samples (note: 100 is an abitrary number, you could go through all of them if you wanted)\n",
- "top_100_wrong = test_df[test_df[\"correct\"] == False].sort_values(\"pred_prob\", ascending=False)[:100]\n",
- "top_100_wrong"
- ],
- "execution_count": null,
"outputs": [
{
- "output_type": "execute_result",
"data": {
"text/html": [
"
\n",
@@ -6166,9 +6142,15 @@
"[100 rows x 7 columns]"
]
},
+ "execution_count": 120,
"metadata": {},
- "execution_count": 120
+ "output_type": "execute_result"
}
+ ],
+ "source": [
+ "# Find top 100 most wrong samples (note: 100 is an abitrary number, you could go through all of them if you wanted)\n",
+ "top_100_wrong = test_df[test_df[\"correct\"] == False].sort_values(\"pred_prob\", ascending=False)[:100]\n",
+ "top_100_wrong"
]
},
{
@@ -6182,24 +6164,17 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
- "id": "ysddyYy717HJ",
"colab": {
"base_uri": "https://localhost:8080/"
},
+ "id": "ysddyYy717HJ",
"outputId": "2a3ad761-8f27-4c5b-b20d-d8169328e3dc"
},
- "source": [
- "# Investigate top wrong preds\n",
- "for row in top_100_wrong[0:10].itertuples(): # adjust indexes to view different samples\n",
- " _, target, text, line_number, total_lines, prediction, pred_prob, _ = row\n",
- " print(f\"Target: {target}, Pred: {prediction}, Prob: {pred_prob}, Line number: {line_number}, Total lines: {total_lines}\\n\")\n",
- " print(f\"Text:\\n{text}\\n\")\n",
- " print(\"-----\\n\")"
- ],
- "execution_count": null,
"outputs": [
{
+ "name": "stdout",
"output_type": "stream",
"text": [
"Target: BACKGROUND, Pred: OBJECTIVE, Prob: 0.9448384046554565, Line number: 0, Total lines: 12\n",
@@ -6272,9 +6247,16 @@
"\n",
"-----\n",
"\n"
- ],
- "name": "stdout"
+ ]
}
+ ],
+ "source": [
+ "# Investigate top wrong preds\n",
+ "for row in top_100_wrong[0:10].itertuples(): # adjust indexes to view different samples\n",
+ " _, target, text, line_number, total_lines, prediction, pred_prob, _ = row\n",
+ " print(f\"Target: {target}, Pred: {prediction}, Prob: {pred_prob}, Line number: {line_number}, Total lines: {total_lines}\\n\")\n",
+ " print(f\"Text:\\n{text}\\n\")\n",
+ " print(\"-----\\n\")"
]
},
{
@@ -6336,26 +6318,18 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
- "id": "-qKFXysU9Y1j",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 414
},
+ "id": "-qKFXysU9Y1j",
"outputId": "7d70562c-90ac-4e71-a020-14ab673bd077"
},
- "source": [
- "# Download and open example abstracts (copy and pasted from PubMed)\n",
- "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/skimlit_example_abstracts.json\n",
- "\n",
- "with open(\"skimlit_example_abstracts.json\", \"r\") as f:\n",
- " example_abstracts = json.load(f)\n",
- "\n",
- "example_abstracts"
- ],
- "execution_count": null,
"outputs": [
{
+ "name": "stdout",
"output_type": "stream",
"text": [
"--2021-08-25 00:08:37-- https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/skimlit_example_abstracts.json\n",
@@ -6369,13 +6343,12 @@
"\n",
"2021-08-25 00:08:37 (82.5 MB/s) - ‘skimlit_example_abstracts.json’ saved [6737/6737]\n",
"\n"
- ],
- "name": "stdout"
+ ]
},
{
- "output_type": "error",
"ename": "NameError",
"evalue": "ignored",
+ "output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
@@ -6383,20 +6356,29 @@
"\u001b[0;31mNameError\u001b[0m: name 'json' is not defined"
]
}
+ ],
+ "source": [
+ "# Download and open example abstracts (copy and pasted from PubMed)\n",
+ "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/skimlit_example_abstracts.json\n",
+ "\n",
+ "with open(\"skimlit_example_abstracts.json\", \"r\") as f:\n",
+ " example_abstracts = json.load(f)\n",
+ "\n",
+ "example_abstracts"
]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "-1cIAS1Z6r_l"
},
+ "outputs": [],
"source": [
"# See what our example abstracts look like\n",
"abstracts = pd.DataFrame(example_abstracts)\n",
"abstracts"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6411,21 +6393,36 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "_gwVNdLQHpQX"
},
+ "outputs": [],
+ "source": [
+ "# # Create sentencizer - Source: https://spacy.io/usage/linguistic-features#sbd \n",
+ "# from spacy.lang.en import English\n",
+ "# nlp = English() # setup English sentence parser\n",
+ "# sentencizer = nlp.create_pipe(\"sentencizer\") # create sentence splitting pipeline object\n",
+ "# nlp.add_pipe(sentencizer) # add sentence splitting pipeline object to sentence parser\n",
+ "# doc = nlp(example_abstracts[0][\"abstract\"]) # create \"doc\" of parsed sequences, change index for a different abstract\n",
+ "# abstract_lines = [str(sent) for sent in list(doc.sents)] # return detected sentences from doc in string type (not spaCy token type)\n",
+ "# abstract_lines"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"# Create sentencizer - Source: https://spacy.io/usage/linguistic-features#sbd \n",
"from spacy.lang.en import English\n",
- "nlp = English() # setup English sentence parser\n",
- "sentencizer = nlp.create_pipe(\"sentencizer\") # create sentence splitting pipeline object\n",
- "nlp.add_pipe(sentencizer) # add sentence splitting pipeline object to sentence parser\n",
- "doc = nlp(example_abstracts[0][\"abstract\"]) # create \"doc\" of parsed sequences, change index for a different abstract\n",
- "abstract_lines = [str(sent) for sent in list(doc.sents)] # return detected sentences from doc in string type (not spaCy token type)\n",
+ "nlp= English()\n",
+ "nlp.add_pipe(\"sentencizer\")\n",
+ "doc= nlp(example_abstracts[0][\"abstract\"])\n",
+ "abstract_lines= [str(sent) for sent in list(doc.sents)]\n",
"abstract_lines"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6442,9 +6439,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "A_Hi0alJI4Xu"
},
+ "outputs": [],
"source": [
"# Get total number of lines\n",
"total_lines_in_sample = len(abstract_lines)\n",
@@ -6458,9 +6457,7 @@
" sample_dict[\"total_lines\"] = total_lines_in_sample - 1\n",
" sample_lines.append(sample_dict)\n",
"sample_lines"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6473,33 +6470,33 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "rm0MYaAnBkbp"
},
+ "outputs": [],
"source": [
"# Get all line_number values from sample abstract\n",
"test_abstract_line_numbers = [line[\"line_number\"] for line in sample_lines]\n",
"# One-hot encode to same depth as training data, so model accepts right input shape\n",
"test_abstract_line_numbers_one_hot = tf.one_hot(test_abstract_line_numbers, depth=15) \n",
"test_abstract_line_numbers_one_hot"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "8Wzbv3w6B3OU"
},
+ "outputs": [],
"source": [
"# Get all total_lines values from sample abstract\n",
"test_abstract_total_lines = [line[\"total_lines\"] for line in sample_lines]\n",
"# One-hot encode to same depth as training data, so model accepts right input shape\n",
"test_abstract_total_lines_one_hot = tf.one_hot(test_abstract_total_lines, depth=20)\n",
"test_abstract_total_lines_one_hot"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6512,16 +6509,16 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "HOOPoG3cCA0F"
},
+ "outputs": [],
"source": [
"# Split abstract lines into characters\n",
"abstract_chars = [split_chars(sentence) for sentence in abstract_lines]\n",
"abstract_chars"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6534,9 +6531,11 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "0b7siZa1CQG7"
},
+ "outputs": [],
"source": [
"# Make predictions on sample abstract features\n",
"%%time\n",
@@ -6545,22 +6544,20 @@
" tf.constant(abstract_lines),\n",
" tf.constant(abstract_chars)))\n",
"test_abstract_pred_probs"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "8nxqfCBfCqWe"
},
+ "outputs": [],
"source": [
"# Turn prediction probabilities into prediction classes\n",
"test_abstract_preds = tf.argmax(test_abstract_pred_probs, axis=1)\n",
"test_abstract_preds"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6573,29 +6570,29 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "LduhApa3C1mD"
},
+ "outputs": [],
"source": [
"# Turn prediction class integers into string class names\n",
"test_abstract_pred_classes = [label_encoder.classes_[i] for i in test_abstract_preds]\n",
"test_abstract_pred_classes"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {
"id": "BhhDPZSHDCJD"
},
+ "outputs": [],
"source": [
"# Visualize abstract lines and predicted sequence labels\n",
"for i, line in enumerate(abstract_lines):\n",
" print(f\"{test_abstract_pred_classes[i]}: {line}\")"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -6656,5 +6653,33 @@
"* Since our dataset has imbalanced classes (as with many real-world datasets), so it might be worth looking into the [TensorFlow guide for different methods to training a model with imbalanced classes](https://www.tensorflow.org/tutorials/structured_data/imbalanced_data).\n"
]
}
- ]
-}
\ No newline at end of file
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "authorship_tag": "ABX9TyM5wpCFGOHhp2o71grvS7VP",
+ "collapsed_sections": [],
+ "include_colab_link": true,
+ "mount_file_id": "1_yq3R-ThKP78_byQV9OovntEcXpQcBK2",
+ "name": "09_SkimLit_nlp_milestone_project_2.ipynb",
+ "provenance": [],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python",
+ "version": "3.10.0"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "1c513e6c7cb82cbcc6920a5ad5b02bc5f0812c20ddf18b6fe3fefee2aa23c21d"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}