From 9b4f83e13b4467ff4ee9d268838eb5e54462c738 Mon Sep 17 00:00:00 2001 From: ravenscroftj Date: Sat, 2 Apr 2022 05:25:44 +0000 Subject: [PATCH] Add 'brainsteam/content/bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142.md' --- ...opping-at-master-tensorflow-models1648877142.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 brainsteam/content/bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142.md diff --git a/brainsteam/content/bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142.md b/brainsteam/content/bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142.md new file mode 100644 index 0000000..7115e1e --- /dev/null +++ b/brainsteam/content/bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142.md @@ -0,0 +1,14 @@ +--- +bookmark-of: https://github.com/tensorflow/models/tree/master/official/projects/token_dropping +date: '2022-04-02T01:25:42.935585' +tags: +- nlp +title: "models/official/projects/token_dropping at master \xB7 tensorflow/models" +type: bookmark +url: /bookmarks/2022/04/02/models-official-projects-token-dropping-at-master-tensorflow-models1648877142 + +--- + +> Token dropping aims to accelerate the pretraining of transformer models such as BERT without degrading its performance on downstream tasks. + +> A BERT model pretrained using this token dropping method is not different to a BERT model pretrained in the conventional way: a BERT checkpoint pretrained with token dropping can be viewed and used as a normal BERT checkpoint, for finetuning etc. \ No newline at end of file