Add app files

.gitignore

@@ -0,0 +1 @@
+node_modules

app.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+import streamlit as st
+import streamlit.components.v1 as components
+import torch
+import torch.nn.functional as F
+from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding
+
+root_dir = Path(__file__).resolve().parent
+_highlighted_text_fn = components.declare_component(
+    "highlighted_text", path=root_dir / "highlighted_text" / "build"
+)
+
+def highlighted_text(tokens, scores, key=None):
+    return _highlighted_text_fn(tokens=tokens, scores=scores, key=key, default=0)
+
+def get_windows_batched(examples: BatchEncoding, window_len: int, stride: int = 1, pad_id: int = 0) -> BatchEncoding:
+    return BatchEncoding({
+        k: [
+            t[i][j : j + window_len] + [
+                pad_id if k == "input_ids" else 0
+            ] * (j + window_len - len(t[i]))
+            for i in range(len(examples["input_ids"]))
+            for j in range(0, len(examples["input_ids"][i]) - 1, stride)
+        ]
+        for k, t in examples.items()
+    })
+
+BAD_CHAR = chr(0xfffd)
+
+def ids_to_readable_tokens(tokenizer, ids, strip_whitespace=False):
+    cur_ids = []
+    result = []
+    for idx in ids:
+        cur_ids.append(idx)
+        decoded = tokenizer.decode(cur_ids)
+        if BAD_CHAR not in decoded:
+            if strip_whitespace:
+                decoded = decoded.strip()
+            result.append(decoded)
+            del cur_ids[:]
+        else:
+            result.append("")
+    return result
+
+model_name = st.selectbox("Model", ["distilgpt2"])
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+window_len = st.select_slider("Window size", options=[8, 16, 32, 64, 128, 256, 512, 1024], value=512)
+text = st.text_area("Input text", "The complex houses married and single soldiers and their families.")
+
+inputs = tokenizer([text])
+[input_ids] = inputs["input_ids"]
+window_len = min(window_len, len(input_ids))
+tokens = ids_to_readable_tokens(tokenizer, input_ids)
+
+inputs_sliding = get_windows_batched(
+    inputs,
+    window_len=window_len,
+    pad_id=tokenizer.eos_token_id
+)
+with torch.inference_mode():
+    logits = model(**inputs_sliding.convert_to_tensors("pt")).logits.to(torch.float16)
+    logits = F.pad(logits, (0, 0, 0, window_len - 1, 0, 0), value=torch.nan)
+    logits = logits.view(-1, logits.shape[-1])[:(window_len - 1) * (len(input_ids) + window_len - 2)]
+    logits = logits.view(window_len - 1, len(input_ids) + window_len - 2, logits.shape[-1])
+
+scores = logits.to(torch.float32).softmax(dim=-1)
+scores = scores[:, torch.arange(len(input_ids[1:])), input_ids[1:]]
+scores = scores.diff(dim=0).transpose(0, 1)
+scores = scores.nan_to_num()
+scores /= scores.abs().max(dim=1, keepdim=True).values + 1e-9
+scores = scores.to(torch.float16)
+print(scores)
+st.markdown("---")
+highlighted_text(tokens=tokens, scores=scores.tolist())
+

highlighted_text/.env

@@ -0,0 +1,6 @@
+# Run the component's dev server on :3001
+# (The Streamlit dev server already runs on :3000)
+PORT=3001
+
+# Don't automatically open the web browser on `npm run start`.
+BROWSER=none

highlighted_text/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "endOfLine": "lf",
+  "semi": false,
+  "trailingComma": "es5"
+}

highlighted_text/build/asset-manifest.json

@@ -0,0 +1,17 @@
+{
+  "files": {
+    "main.js": "./static/js/main.243f5634.chunk.js",
+    "main.js.map": "./static/js/main.243f5634.chunk.js.map",
+    "runtime-main.js": "./static/js/runtime-main.e6b0ae4c.js",
+    "runtime-main.js.map": "./static/js/runtime-main.e6b0ae4c.js.map",
+    "static/js/2.ce130e37.chunk.js": "./static/js/2.ce130e37.chunk.js",
+    "static/js/2.ce130e37.chunk.js.map": "./static/js/2.ce130e37.chunk.js.map",
+    "index.html": "./index.html",
+    "static/js/2.ce130e37.chunk.js.LICENSE.txt": "./static/js/2.ce130e37.chunk.js.LICENSE.txt"
+  },
+  "entrypoints": [
+    "static/js/runtime-main.e6b0ae4c.js",
+    "static/js/2.ce130e37.chunk.js",
+    "static/js/main.243f5634.chunk.js"
+  ]
+}

highlighted_text/build/index.css

@@ -0,0 +1,36 @@
+body {
+    padding: 0;
+    margin: 0;
+    font-family: "Source Sans Pro", sans-serif;
+    font-size: 1rem;
+    line-height: 1.4;
+}
+
+.status-bar {
+    display: none;
+}
+
+.highlighted-text {
+    border: 1px solid #d2d2d2;
+    border-radius: 5px 5px 5px 5px;
+    padding: 2px;
+    cursor: pointer;
+
+    .token.active {
+        outline: 1px solid #444;
+    }
+
+    &.frozen {
+        .token {
+            opacity: 0.75;
+
+            &.context, &.active {
+                opacity: 1;
+            }
+
+            &.context {
+                text-decoration: #999 underline;
+            }
+        }
+    }
+}

highlighted_text/build/index.html

@@ -0,0 +1 @@
+<!doctype html><html lang="en"><head><title>Highlighted text component</title><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#fff"/><meta name="description" content="Highlighted text"/><link rel="stylesheet" href="index.css"/></head><body><div id="root"></div><script>!function(e){function t(t){for(var n,l,a=t[0],p=t[1],i=t[2],c=0,s=[];c<a.length;c++)l=a[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in p)Object.prototype.hasOwnProperty.call(p,n)&&(e[n]=p[n]);for(f&&f(t);s.length;)s.shift()();return u.push.apply(u,i||[]),r()}function r(){for(var e,t=0;t<u.length;t++){for(var r=u[t],n=!0,a=1;a<r.length;a++){var p=r[a];0!==o[p]&&(n=!1)}n&&(u.splice(t--,1),e=l(l.s=r[0]))}return e}var n={},o={1:0},u=[];function l(t){if(n[t])return n[t].exports;var r=n[t]={i:t,l:!1,exports:{}};return e[t].call(r.exports,r,r.exports,l),r.l=!0,r.exports}l.m=e,l.c=n,l.d=function(e,t,r){l.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},l.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,t){if(1&t&&(e=l(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(l.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var n in e)l.d(r,n,function(t){return e[t]}.bind(null,n));return r},l.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(t,"a",t),t},l.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},l.p="./";var a=this.webpackJsonpstreamlit_component_template=this.webpackJsonpstreamlit_component_template||[],p=a.push.bind(a);a.push=t,a=a.slice();for(var i=0;i<a.length;i++)t(a[i]);var f=p;r()}([])</script><script src="./static/js/2.ce130e37.chunk.js"></script><script src="./static/js/main.243f5634.chunk.js"></script></body></html>

highlighted_text/build/static/js/2.ce130e37.chunk.js.LICENSE.txt

@@ -0,0 +1,69 @@
+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+
+/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
+
+/**
+ * @license
+ * Copyright 2018-2021 Streamlit Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** @license React v0.19.1
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.13.1
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.14.0
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.14.0
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/** @license React v16.14.0
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

highlighted_text/build/static/js/main.243f5634.chunk.js

@@ -0,0 +1,2 @@
+(this.webpackJsonpstreamlit_component_template=this.webpackJsonpstreamlit_component_template||[]).push([[0],{27:function(t,e,a){"use strict";a.r(e);var n=a(7),s=a.n(n),r=a(18),c=a.n(r),i=a(4),o=a(0),l=a(1),h=a(2),j=a(3),u=a(16),v=a(6),b=function(t){Object(h.a)(a,t);var e=Object(j.a)(a);function a(){var t;Object(o.a)(this,a);for(var n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))).state={activeIndex:null,hoverIndex:null,isFrozen:!1},t}return Object(l.a)(a,[{key:"render",value:function(){var t=this,e=this.props.args.tokens,a=this.getScores();console.log(a);var n="highlighted-text";this.state.isFrozen&&(n+=" frozen");var s=function(){t.setState({isFrozen:!1})};return Object(v.jsxs)(v.Fragment,{children:[Object(v.jsxs)("div",{className:"status-bar",children:[Object(v.jsxs)("span",{className:this.state.isFrozen?"":" d-none",children:[Object(v.jsx)("i",{className:"fa fa-lock"})," "]},"lock-icon"),Object(v.jsx)("strong",{children:"target:"},"target-label"),null!=this.state.activeIndex?Object(v.jsx)("span",{className:"token",children:e[this.state.activeIndex]},"target"):Object(v.jsx)(v.Fragment,{})]},"status-bar"),Object(v.jsx)("div",{className:n,onClick:s,children:e.map((function(e,n){var r="token";t.state&&t.state.activeIndex==n&&(r+=" active");var c={backgroundColor:a[n]>0?"rgba(255, 32, 32, ".concat(a[n],")"):"rgba(32, 255, 32, ".concat(-a[n],")")};return Object(v.jsx)("span",{className:r,style:c,onMouseOver:function(){t.state.isFrozen||t.setState({activeIndex:n}),t.setState({hoverIndex:n})},onClick:s,children:e},n)}))},"text")]})}},{key:"getScores",value:function(){var t=this.props.args.tokens;if(!this.state||null==this.state.activeIndex||this.state.activeIndex<1)return t.map((function(){return 0}));var e=this.props.args.scores,a=this.state.activeIndex-1,n=Math.min(Math.max(0,a),e[a].length),s=e[a].slice(0,n);s.reverse();var r=[].concat(Object(i.a)(Array(Math.max(0,a-1-s.length)).fill(0)),Object(i.a)(s.map((function(t){return void 0==t||isNaN(t)?0:t}))));return r=[].concat(Object(i.a)(r),Object(i.a)(Array(t.length-r.length).fill(0)))}}]),a}(u.a),d=Object(u.b)(b);c.a.render(Object(v.jsx)(s.a.StrictMode,{children:Object(v.jsx)(d,{})}),document.getElementById("root"))}},[[27,1,2]]]);
+//# sourceMappingURL=main.243f5634.chunk.js.map

highlighted_text/build/static/js/main.243f5634.chunk.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["HighlightedText.tsx","index.tsx"],"names":["HighlightedText","_StreamlitComponentBa","_inherits","_super","_createSuper","_this","_classCallCheck","_len","arguments","length","args","Array","_key","call","apply","concat","state","activeIndex","hoverIndex","isFrozen","_createClass","key","value","_this2","tokens","this","props","scores","getScores","console","log","className","onClick","setState","_jsxs","_Fragment","children","_jsx","map","t","i","style","backgroundColor","onMouseOver","allScores","hi","Math","min","max","row","slice","reverse","result","_toConsumableArray","fill","x","undefined","isNaN","StreamlitComponentBase","withStreamlitConnection","ReactDOM","render","React","StrictMode","document","getElementById"],"mappings":"2OAiBMA,EAAe,SAAAC,GAAAC,YAAAF,EAAAC,GAAA,IAAAE,EAAAC,YAAAJ,GAAA,SAAAA,IAAA,IAAAK,EAAAC,YAAA,KAAAN,GAAA,QAAAO,EAAAC,UAAAC,OAAAC,EAAA,IAAAC,MAAAJ,GAAAK,EAAA,EAAAA,EAAAL,EAAAK,IAAAF,EAAAE,GAAAJ,UAAAI,GACqD,OADrDP,EAAAF,EAAAU,KAAAC,MAAAX,EAAA,OAAAY,OAAAL,KACVM,MAAQ,CAACC,YAAa,KAAMC,WAAY,KAAMC,UAAU,GAAMd,CAAC,CAyErE,OAzEoEe,YAAApB,EAAA,EAAAqB,IAAA,SAAAC,MAErE,WAAU,IAADC,EAAA,KACCC,EAAmBC,KAAKC,MAAMhB,KAAa,OAC3CiB,EAAmBF,KAAKG,YAC9BC,QAAQC,IAAIH,GAEZ,IAAII,EAAY,mBACZN,KAAKT,MAAMG,WACXY,GAAa,WAGjB,IAAMC,EAAU,WACZT,EAAKU,SAAS,CAAEd,UAAU,GAC9B,EAEA,OAAOe,eAAAC,WAAA,CAAAC,SAAA,CACHF,eAAA,OAAKH,UAAU,aAAYK,SAAA,CACvBF,eAAA,QAAMH,UAAWN,KAAKT,MAAMG,SAAW,GAAK,UAAUiB,SAAA,CAAiBC,cAAA,KAAGN,UAAU,eAAiB,MAA1C,aAC3DM,cAAA,UAAAD,SAA2B,WAAf,gBAEkB,MAA1BX,KAAKT,MAAMC,YACLoB,cAAA,QAAMN,UAAU,QAAOK,SAAeZ,EAAOC,KAAKT,MAAMC,cAA5B,UAC5BoB,cAAAF,WAAA,MANkB,cAShCE,cAAA,OAAKN,UAAWA,EAAWC,QAASA,EAAQI,SAEpCZ,EAAOc,KAAI,SAACC,EAAWC,GACnB,IAAIT,EAAY,QACZR,EAAKP,OACDO,EAAKP,MAAMC,aAAeuB,IAC1BT,GAAa,WAGrB,IAAMU,EAAQ,CACVC,gBACIf,EAAOa,GAAK,EAAC,qBAAAzB,OACcY,EAAOa,GAAE,0BAAAzB,QACRY,EAAOa,GAAE,MAS7C,OAAOH,cAAA,QAAcN,UAAWA,EAAWU,MAAOA,EAC9CE,YAPgB,WACXpB,EAAKP,MAAMG,UACZI,EAAKU,SAAS,CAAEhB,YAAauB,IAEjCjB,EAAKU,SAAS,CAAEf,WAAYsB,GAChC,EAE8BR,QAASA,EAAQI,SAAEG,GAD/BC,EAEtB,KAxByC,UA4BzD,GAAC,CAAAnB,IAAA,YAAAC,MAED,WACI,IAAME,EAASC,KAAKC,MAAMhB,KAAa,OACvC,IAAKe,KAAKT,OAAmC,MAA1BS,KAAKT,MAAMC,aAAuBQ,KAAKT,MAAMC,YAAc,EAC1E,OAAOO,EAAOc,KAAI,kBAAM,CAAC,IAE7B,IAAMM,EAAwBnB,KAAKC,MAAMhB,KAAa,OAEhD8B,EAAIf,KAAKT,MAAMC,YAAc,EAC7B4B,EAAKC,KAAKC,IAAID,KAAKE,IAAI,EAAGR,GAAII,EAAUJ,GAAG/B,QAC3CwC,EAAML,EAAUJ,GAAGU,MAAM,EAAGL,GAClCI,EAAIE,UACJ,IAAIC,EAAM,GAAArC,OAAAsC,YACH1C,MAAMmC,KAAKE,IAAI,EAAGR,EAAI,EAAIS,EAAIxC,SAAS6C,KAAK,IAAED,YAC9CJ,EAAIX,KAAI,SAACiB,GAAC,YAAUC,GAALD,GAAkBE,MAAMF,GAAK,EAAIA,CAAC,MAGxD,OADAH,EAAM,GAAArC,OAAAsC,YAAOD,GAAMC,YAAK1C,MAAMa,EAAOf,OAAS2C,EAAO3C,QAAQ6C,KAAK,IAEtE,KAACtD,CAAA,CA1EgB,CAAS0D,KA6EfC,cAAwB3D,GC1FvC4D,IAASC,OACPxB,cAACyB,IAAMC,WAAU,CAAA3B,SACfC,cAACrC,EAAe,MAElBgE,SAASC,eAAe,Q","file":"static/js/main.243f5634.chunk.js","sourcesContent":["import {\n    Streamlit,\n    StreamlitComponentBase,\n    withStreamlitConnection,\n} from \"streamlit-component-lib\";\nimport React, { ReactNode } from \"react\";\n\ntype HighlightedTextState = {\n    activeIndex: number | null,\n    hoverIndex: number | null,\n    isFrozen: boolean\n};\n\n/**\n * This is a React-based component template. The `render()` function is called\n * automatically when your component should be re-rendered.\n */\nclass HighlightedText extends StreamlitComponentBase<HighlightedTextState> {\n    public state = {activeIndex: null, hoverIndex: null, isFrozen: false};\n\n    render() {\n        const tokens: string[] = this.props.args[\"tokens\"];\n        const scores: number[] = this.getScores();\n        console.log(scores);\n\n        let className = \"highlighted-text\";\n        if (this.state.isFrozen) {\n            className += \" frozen\";\n        }\n\n        const onClick = () => {\n            this.setState({ isFrozen: false });\n        };\n\n        return <>\n            <div className=\"status-bar\" key=\"status-bar\">\n                <span className={this.state.isFrozen ? \"\" : \" d-none\"} key=\"lock-icon\"><i className=\"fa fa-lock\"></i> </span>\n                <strong key=\"target-label\">target:</strong>\n                {\n                    this.state.activeIndex != null\n                        ? <span className=\"token\" key=\"target\">{tokens[this.state.activeIndex]}</span>\n                        : <></>\n                }\n            </div>\n            <div className={className} onClick={onClick} key=\"text\">\n                {\n                    tokens.map((t: string, i: number) => {\n                        let className = \"token\";\n                        if (this.state) {\n                            if (this.state.activeIndex == i) {\n                                className += \" active\";\n                            }\n                        }\n                        const style = {\n                            backgroundColor:\n                                scores[i] > 0\n                                    ? `rgba(255, 32, 32, ${scores[i]})`\n                                    : `rgba(32, 255, 32, ${-scores[i]})`\n                        };\n\n                        const onMouseOver = () => {\n                            if (!this.state.isFrozen) {\n                                this.setState({ activeIndex: i });\n                            }\n                            this.setState({ hoverIndex: i });\n                        };\n                        return <span key={i} className={className} style={style}\n                            onMouseOver={onMouseOver} onClick={onClick}>{t}</span>;\n                    })\n                }\n            </div>\n        </>;\n    }\n\n    private getScores() {\n        const tokens = this.props.args[\"tokens\"];\n        if (!this.state || this.state.activeIndex == null || this.state.activeIndex < 1) {\n            return tokens.map(() => 0);\n        }\n        const allScores: number[][] = this.props.args[\"scores\"];\n\n        const i = this.state.activeIndex - 1;\n        const hi = Math.min(Math.max(0, i), allScores[i].length);\n        const row = allScores[i].slice(0, hi);\n        row.reverse();\n        let result = [\n            ...Array(Math.max(0, i - 1 - row.length)).fill(0), \n            ...row.map((x) => x == undefined || isNaN(x) ? 0 : x)\n        ];\n        result = [...result, ...Array(tokens.length - result.length).fill(0)];\n        return result;\n    }\n}\n\nexport default withStreamlitConnection(HighlightedText);\n","import React from \"react\";\nimport ReactDOM from \"react-dom\";\nimport HighlightedText from \"./HighlightedText\";\n\nReactDOM.render(\n  <React.StrictMode>\n    <HighlightedText />\n  </React.StrictMode>,\n  document.getElementById(\"root\")\n)\n"],"sourceRoot":""}

highlighted_text/build/static/js/runtime-main.e6b0ae4c.js

@@ -0,0 +1,2 @@
+!function(e){function t(t){for(var n,l,a=t[0],p=t[1],i=t[2],c=0,s=[];c<a.length;c++)l=a[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in p)Object.prototype.hasOwnProperty.call(p,n)&&(e[n]=p[n]);for(f&&f(t);s.length;)s.shift()();return u.push.apply(u,i||[]),r()}function r(){for(var e,t=0;t<u.length;t++){for(var r=u[t],n=!0,a=1;a<r.length;a++){var p=r[a];0!==o[p]&&(n=!1)}n&&(u.splice(t--,1),e=l(l.s=r[0]))}return e}var n={},o={1:0},u=[];function l(t){if(n[t])return n[t].exports;var r=n[t]={i:t,l:!1,exports:{}};return e[t].call(r.exports,r,r.exports,l),r.l=!0,r.exports}l.m=e,l.c=n,l.d=function(e,t,r){l.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},l.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,t){if(1&t&&(e=l(e)),8&t)return e;if(4&t&&"object"===typeof e&&e&&e.__esModule)return e;var r=Object.create(null);if(l.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var n in e)l.d(r,n,function(t){return e[t]}.bind(null,n));return r},l.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(t,"a",t),t},l.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},l.p="./";var a=this.webpackJsonpstreamlit_component_template=this.webpackJsonpstreamlit_component_template||[],p=a.push.bind(a);a.push=t,a=a.slice();for(var i=0;i<a.length;i++)t(a[i]);var f=p;r()}([]);
+//# sourceMappingURL=runtime-main.e6b0ae4c.js.map

highlighted_text/build/static/js/runtime-main.e6b0ae4c.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../webpack/bootstrap"],"names":["webpackJsonpCallback","data","moduleId","chunkId","chunkIds","moreModules","executeModules","i","resolves","length","Object","prototype","hasOwnProperty","call","installedChunks","push","modules","parentJsonpFunction","shift","deferredModules","apply","checkDeferredModules","result","deferredModule","fulfilled","j","depId","splice","__webpack_require__","s","installedModules","exports","module","l","m","c","d","name","getter","o","defineProperty","enumerable","get","r","Symbol","toStringTag","value","t","mode","__esModule","ns","create","key","bind","n","object","property","p","jsonpArray","this","oldJsonpFunction","slice"],"mappings":"aACE,SAASA,EAAqBC,GAQ7B,IAPA,IAMIC,EAAUC,EANVC,EAAWH,EAAK,GAChBI,EAAcJ,EAAK,GACnBK,EAAiBL,EAAK,GAIHM,EAAI,EAAGC,EAAW,GACpCD,EAAIH,EAASK,OAAQF,IACzBJ,EAAUC,EAASG,GAChBG,OAAOC,UAAUC,eAAeC,KAAKC,EAAiBX,IAAYW,EAAgBX,IACpFK,EAASO,KAAKD,EAAgBX,GAAS,IAExCW,EAAgBX,GAAW,EAE5B,IAAID,KAAYG,EACZK,OAAOC,UAAUC,eAAeC,KAAKR,EAAaH,KACpDc,EAAQd,GAAYG,EAAYH,IAKlC,IAFGe,GAAqBA,EAAoBhB,GAEtCO,EAASC,QACdD,EAASU,OAATV,GAOD,OAHAW,EAAgBJ,KAAKK,MAAMD,EAAiBb,GAAkB,IAGvDe,GACR,CACA,SAASA,IAER,IADA,IAAIC,EACIf,EAAI,EAAGA,EAAIY,EAAgBV,OAAQF,IAAK,CAG/C,IAFA,IAAIgB,EAAiBJ,EAAgBZ,GACjCiB,GAAY,EACRC,EAAI,EAAGA,EAAIF,EAAed,OAAQgB,IAAK,CAC9C,IAAIC,EAAQH,EAAeE,GACG,IAA3BX,EAAgBY,KAAcF,GAAY,EAC9C,CACGA,IACFL,EAAgBQ,OAAOpB,IAAK,GAC5Be,EAASM,EAAoBA,EAAoBC,EAAIN,EAAe,IAEtE,CAEA,OAAOD,CACR,CAGA,IAAIQ,EAAmB,CAAC,EAKpBhB,EAAkB,CACrB,EAAG,GAGAK,EAAkB,GAGtB,SAASS,EAAoB1B,GAG5B,GAAG4B,EAAiB5B,GACnB,OAAO4B,EAAiB5B,GAAU6B,QAGnC,IAAIC,EAASF,EAAiB5B,GAAY,CACzCK,EAAGL,EACH+B,GAAG,EACHF,QAAS,CAAC,GAUX,OANAf,EAAQd,GAAUW,KAAKmB,EAAOD,QAASC,EAAQA,EAAOD,QAASH,GAG/DI,EAAOC,GAAI,EAGJD,EAAOD,OACf,CAIAH,EAAoBM,EAAIlB,EAGxBY,EAAoBO,EAAIL,EAGxBF,EAAoBQ,EAAI,SAASL,EAASM,EAAMC,GAC3CV,EAAoBW,EAAER,EAASM,IAClC3B,OAAO8B,eAAeT,EAASM,EAAM,CAAEI,YAAY,EAAMC,IAAKJ,GAEhE,EAGAV,EAAoBe,EAAI,SAASZ,GACX,qBAAXa,QAA0BA,OAAOC,aAC1CnC,OAAO8B,eAAeT,EAASa,OAAOC,YAAa,CAAEC,MAAO,WAE7DpC,OAAO8B,eAAeT,EAAS,aAAc,CAAEe,OAAO,GACvD,EAOAlB,EAAoBmB,EAAI,SAASD,EAAOE,GAEvC,GADU,EAAPA,IAAUF,EAAQlB,EAAoBkB,IAC/B,EAAPE,EAAU,OAAOF,EACpB,GAAW,EAAPE,GAA8B,kBAAVF,GAAsBA,GAASA,EAAMG,WAAY,OAAOH,EAChF,IAAII,EAAKxC,OAAOyC,OAAO,MAGvB,GAFAvB,EAAoBe,EAAEO,GACtBxC,OAAO8B,eAAeU,EAAI,UAAW,CAAET,YAAY,EAAMK,MAAOA,IACtD,EAAPE,GAA4B,iBAATF,EAAmB,IAAI,IAAIM,KAAON,EAAOlB,EAAoBQ,EAAEc,EAAIE,EAAK,SAASA,GAAO,OAAON,EAAMM,EAAM,EAAEC,KAAK,KAAMD,IAC9I,OAAOF,CACR,EAGAtB,EAAoB0B,EAAI,SAAStB,GAChC,IAAIM,EAASN,GAAUA,EAAOiB,WAC7B,WAAwB,OAAOjB,EAAgB,OAAG,EAClD,WAA8B,OAAOA,CAAQ,EAE9C,OADAJ,EAAoBQ,EAAEE,EAAQ,IAAKA,GAC5BA,CACR,EAGAV,EAAoBW,EAAI,SAASgB,EAAQC,GAAY,OAAO9C,OAAOC,UAAUC,eAAeC,KAAK0C,EAAQC,EAAW,EAGpH5B,EAAoB6B,EAAI,KAExB,IAAIC,EAAaC,KAA+C,yCAAIA,KAA+C,0CAAK,GACpHC,EAAmBF,EAAW3C,KAAKsC,KAAKK,GAC5CA,EAAW3C,KAAOf,EAClB0D,EAAaA,EAAWG,QACxB,IAAI,IAAItD,EAAI,EAAGA,EAAImD,EAAWjD,OAAQF,IAAKP,EAAqB0D,EAAWnD,IAC3E,IAAIU,EAAsB2C,EAI1BvC,G","file":"static/js/runtime-main.e6b0ae4c.js","sourcesContent":[" \t// install a JSONP callback for chunk loading\n \tfunction webpackJsonpCallback(data) {\n \t\tvar chunkIds = data[0];\n \t\tvar moreModules = data[1];\n \t\tvar executeModules = data[2];\n\n \t\t// add \"moreModules\" to the modules object,\n \t\t// then flag all \"chunkIds\" as loaded and fire callback\n \t\tvar moduleId, chunkId, i = 0, resolves = [];\n \t\tfor(;i < chunkIds.length; i++) {\n \t\t\tchunkId = chunkIds[i];\n \t\t\tif(Object.prototype.hasOwnProperty.call(installedChunks, chunkId) && installedChunks[chunkId]) {\n \t\t\t\tresolves.push(installedChunks[chunkId][0]);\n \t\t\t}\n \t\t\tinstalledChunks[chunkId] = 0;\n \t\t}\n \t\tfor(moduleId in moreModules) {\n \t\t\tif(Object.prototype.hasOwnProperty.call(moreModules, moduleId)) {\n \t\t\t\tmodules[moduleId] = moreModules[moduleId];\n \t\t\t}\n \t\t}\n \t\tif(parentJsonpFunction) parentJsonpFunction(data);\n\n \t\twhile(resolves.length) {\n \t\t\tresolves.shift()();\n \t\t}\n\n \t\t// add entry modules from loaded chunk to deferred list\n \t\tdeferredModules.push.apply(deferredModules, executeModules || []);\n\n \t\t// run deferred modules when all chunks ready\n \t\treturn checkDeferredModules();\n \t};\n \tfunction checkDeferredModules() {\n \t\tvar result;\n \t\tfor(var i = 0; i < deferredModules.length; i++) {\n \t\t\tvar deferredModule = deferredModules[i];\n \t\t\tvar fulfilled = true;\n \t\t\tfor(var j = 1; j < deferredModule.length; j++) {\n \t\t\t\tvar depId = deferredModule[j];\n \t\t\t\tif(installedChunks[depId] !== 0) fulfilled = false;\n \t\t\t}\n \t\t\tif(fulfilled) {\n \t\t\t\tdeferredModules.splice(i--, 1);\n \t\t\t\tresult = __webpack_require__(__webpack_require__.s = deferredModule[0]);\n \t\t\t}\n \t\t}\n\n \t\treturn result;\n \t}\n\n \t// The module cache\n \tvar installedModules = {};\n\n \t// object to store loaded and loading chunks\n \t// undefined = chunk not loaded, null = chunk preloaded/prefetched\n \t// Promise = chunk loading, 0 = chunk loaded\n \tvar installedChunks = {\n \t\t1: 0\n \t};\n\n \tvar deferredModules = [];\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId]) {\n \t\t\treturn installedModules[moduleId].exports;\n \t\t}\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\ti: moduleId,\n \t\t\tl: false,\n \t\t\texports: {}\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.l = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// define getter function for harmony exports\n \t__webpack_require__.d = function(exports, name, getter) {\n \t\tif(!__webpack_require__.o(exports, name)) {\n \t\t\tObject.defineProperty(exports, name, { enumerable: true, get: getter });\n \t\t}\n \t};\n\n \t// define __esModule on exports\n \t__webpack_require__.r = function(exports) {\n \t\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n \t\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n \t\t}\n \t\tObject.defineProperty(exports, '__esModule', { value: true });\n \t};\n\n \t// create a fake namespace object\n \t// mode & 1: value is a module id, require it\n \t// mode & 2: merge all properties of value into the ns\n \t// mode & 4: return value when already ns object\n \t// mode & 8|1: behave like require\n \t__webpack_require__.t = function(value, mode) {\n \t\tif(mode & 1) value = __webpack_require__(value);\n \t\tif(mode & 8) return value;\n \t\tif((mode & 4) && typeof value === 'object' && value && value.__esModule) return value;\n \t\tvar ns = Object.create(null);\n \t\t__webpack_require__.r(ns);\n \t\tObject.defineProperty(ns, 'default', { enumerable: true, value: value });\n \t\tif(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key));\n \t\treturn ns;\n \t};\n\n \t// getDefaultExport function for compatibility with non-harmony modules\n \t__webpack_require__.n = function(module) {\n \t\tvar getter = module && module.__esModule ?\n \t\t\tfunction getDefault() { return module['default']; } :\n \t\t\tfunction getModuleExports() { return module; };\n \t\t__webpack_require__.d(getter, 'a', getter);\n \t\treturn getter;\n \t};\n\n \t// Object.prototype.hasOwnProperty.call\n \t__webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); };\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"./\";\n\n \tvar jsonpArray = this[\"webpackJsonpstreamlit_component_template\"] = this[\"webpackJsonpstreamlit_component_template\"] || [];\n \tvar oldJsonpFunction = jsonpArray.push.bind(jsonpArray);\n \tjsonpArray.push = webpackJsonpCallback;\n \tjsonpArray = jsonpArray.slice();\n \tfor(var i = 0; i < jsonpArray.length; i++) webpackJsonpCallback(jsonpArray[i]);\n \tvar parentJsonpFunction = oldJsonpFunction;\n\n\n \t// run deferred modules from other chunks\n \tcheckDeferredModules();\n"],"sourceRoot":""}

highlighted_text/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "streamlit_component_template",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "react": "^16.13.1",
+    "react-dom": "^16.13.1",
+    "streamlit-component-lib": "^1.3.0"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject"
+  },
+  "eslintConfig": {
+    "extends": "react-app"
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "homepage": ".",
+  "devDependencies": {
+    "@types/node": "^12.0.0",
+    "@types/react": "^16.9.0",
+    "@types/react-dom": "^16.9.0",
+    "react-scripts": "4.0.3",
+    "typescript": "^4.2.0"
+  }
+}

highlighted_text/public/index.css

@@ -0,0 +1,36 @@
+body {
+    padding: 0;
+    margin: 0;
+    font-family: "Source Sans Pro", sans-serif;
+    font-size: 1rem;
+    line-height: 1.4;
+}
+
+.status-bar {
+    display: none;
+}
+
+.highlighted-text {
+    border: 1px solid #d2d2d2;
+    border-radius: 5px 5px 5px 5px;
+    padding: 2px;
+    cursor: pointer;
+
+    .token.active {
+        outline: 1px solid #444;
+    }
+
+    &.frozen {
+        .token {
+            opacity: 0.75;
+
+            &.context, &.active {
+                opacity: 1;
+            }
+
+            &.context {
+                text-decoration: #999 underline;
+            }
+        }
+    }
+}

highlighted_text/public/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>Highlighted text component</title>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="theme-color" content="#fff" />
+    <meta name="description" content="Highlighted text" />
+    <link rel="stylesheet" href="index.css" />
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

highlighted_text/src/HighlightedText.tsx

@@ -0,0 +1,93 @@
+import {
+    StreamlitComponentBase,
+    withStreamlitConnection,
+} from "streamlit-component-lib";
+
+type HighlightedTextState = {
+    activeIndex: number | null,
+    hoverIndex: number | null,
+    isFrozen: boolean
+};
+
+/**
+ * This is a React-based component template. The `render()` function is called
+ * automatically when your component should be re-rendered.
+ */
+class HighlightedText extends StreamlitComponentBase<HighlightedTextState> {
+    public state = {activeIndex: null, hoverIndex: null, isFrozen: false};
+
+    render() {
+        const tokens: string[] = this.props.args["tokens"];
+        const scores: number[] = this.getScores();
+        console.log(scores);
+
+        let className = "highlighted-text";
+        if (this.state.isFrozen) {
+            className += " frozen";
+        }
+
+        const onClick = () => {
+            this.setState({ isFrozen: false });
+        };
+
+        return <>
+            <div className="status-bar" key="status-bar">
+                <span className={this.state.isFrozen ? "" : " d-none"} key="lock-icon"><i className="fa fa-lock"></i> </span>
+                <strong key="target-label">target:</strong>
+                {
+                    this.state.activeIndex != null
+                        ? <span className="token" key="target">{tokens[this.state.activeIndex]}</span>
+                        : <></>
+                }
+            </div>
+            <div className={className} onClick={onClick} key="text">
+                {
+                    tokens.map((t: string, i: number) => {
+                        let className = "token";
+                        if (this.state) {
+                            if (this.state.activeIndex == i) {
+                                className += " active";
+                            }
+                        }
+                        const style = {
+                            backgroundColor:
+                                scores[i] > 0
+                                    ? `rgba(255, 32, 32, ${scores[i]})`
+                                    : `rgba(32, 255, 32, ${-scores[i]})`
+                        };
+
+                        const onMouseOver = () => {
+                            if (!this.state.isFrozen) {
+                                this.setState({ activeIndex: i });
+                            }
+                            this.setState({ hoverIndex: i });
+                        };
+                        return <span key={i} className={className} style={style}
+                            onMouseOver={onMouseOver} onClick={onClick}>{t}</span>;
+                    })
+                }
+            </div>
+        </>;
+    }
+
+    private getScores() {
+        const tokens = this.props.args["tokens"];
+        if (!this.state || this.state.activeIndex == null || this.state.activeIndex < 1) {
+            return tokens.map(() => 0);
+        }
+        const allScores: number[][] = this.props.args["scores"];
+
+        const i = this.state.activeIndex - 1;
+        const hi = Math.min(Math.max(0, i), allScores[i].length);
+        const row = allScores[i].slice(0, hi);
+        row.reverse();
+        let result = [
+            ...Array(Math.max(0, i - 1 - row.length)).fill(0), 
+            ...row.map((x) => x == undefined || isNaN(x) ? 0 : x)
+        ];
+        result = [...result, ...Array(tokens.length - result.length).fill(0)];
+        return result;
+    }
+}
+
+export default withStreamlitConnection(HighlightedText);

highlighted_text/src/index.tsx

@@ -0,0 +1,10 @@
+import React from "react";
+import ReactDOM from "react-dom";
+import HighlightedText from "./HighlightedText";
+
+ReactDOM.render(
+  <React.StrictMode>
+    <HighlightedText />
+  </React.StrictMode>,
+  document.getElementById("root")
+)

highlighted_text/src/react-app-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="react-scripts" />

highlighted_text/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": [
+    "src"
+  ]
+}

requirements.txt

@@ -0,0 +1,3 @@
+streamlit==1.17.0
+torch
+transformers