Push

.env.demo

@@ -1,4 +1,4 @@
 LILAC_DATA_PATH='/data'
 HF_HOME='/data/.huggingface'
-HF_DATASETS_CACHE='/data/.cache'
 TRANSFORMERS_CACHE='/data/.cache'
+XDG_CACHE_HOME='/data/.cache'

.gitattributes

@@ -1,7 +0,0 @@
-data/datasets/local/spotify/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/local/spotify/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/local/spotify/text/sbert/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/local/spotify/text/sbert/embedding/local/outerspace/v34/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.keys.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.npy filter=lfs diff=lfs merge=lfs -text
-data/concept/lilac/profanity/openai.pkl filter=lfs diff=lfs merge=lfs -text

data/concept/lilac/legal-termination/concept.json

@@ -1,184 +0,0 @@
-{
-  "namespace": "lilac",
-  "concept_name": "legal-termination",
-  "type": "text",
-  "data": {
-    "731b1338cf1949958c3526c555f88058": {
-      "label": true,
-      "text": "In the event that any provision of this agreement is found to be unenforceable, the remaining provisions shall continue to be valid and binding.",
-      "id": "731b1338cf1949958c3526c555f88058"
-    },
-    "99a20e547e38474dbc24507a1658d0c9": {
-      "label": true,
-      "text": "The parties agree that in the event of a natural disaster or other unforeseen event, both parties will make reasonable efforts to fulfill their obligations under this contract.",
-      "id": "99a20e547e38474dbc24507a1658d0c9"
-    },
-    "3f27b47c526a4c5896a0a100024535c7": {
-      "label": true,
-      "text": "If any party breaches the terms of this agreement, the non-breaching party shall have the right to seek legal remedies.",
-      "id": "3f27b47c526a4c5896a0a100024535c7"
-    },
-    "d403dbb1ab9c4594bc7f7dcb0ad5b333": {
-      "label": true,
-      "text": "This lease agreement shall survive the termination or expiration of the lease term, and continue to be binding upon the parties.",
-      "id": "d403dbb1ab9c4594bc7f7dcb0ad5b333"
-    },
-    "b7deba9f7e80444abe14448f53f45c43": {
-      "label": true,
-      "text": "In the event of a dispute arising from this contract, the parties agree to first attempt to resolve the dispute through mediation before pursuing any legal action.",
-      "id": "b7deba9f7e80444abe14448f53f45c43"
-    },
-    "a82231b490174e62aad733cb0c75024d": {
-      "label": true,
-      "text": "This Agreement may be terminated, and the transactions contemplated hereby may be abandoned, at any time prior to the Effective Time, whether prior to or after the Company Stockholders' Approval:",
-      "id": "a82231b490174e62aad733cb0c75024d"
-    },
-    "160b25dbf14e4759a0065bbd652ce33f": {
-      "label": true,
-      "text": "This Agreement may be terminated and abandoned at any time prior to the Effective Time of the Merger, whether before or after the Company Stockholder Approval:",
-      "id": "160b25dbf14e4759a0065bbd652ce33f"
-    },
-    "8f5f9f96b16441228bb0c9b8a14c4e25": {
-      "label": false,
-      "text": "any jurisdiction, then such provision shall, as to such jurisdiction, be modified or restricted to the extent necessary to make such provision valid, binding and enforceable, or if such provision cannot be so modified or restricted, then such provision shall, as to such jurisdiction, be deemed to be excised from this Agreement; provided, however, that the legality, binding effect and",
-      "id": "8f5f9f96b16441228bb0c9b8a14c4e25"
-    },
-    "87b6c31b04a346b4a3e0da8d2cc5a7ac": {
-      "label": true,
-      "text": "This Agreement shall terminate automatically without any further action by any party hereto upon the earliest to occur of (a) the Effective Time of the Merger, (b) the termination of the Merger Agreement in accordance with its terms and (c) any amendment or other modification of the Merger Agreement that reduces the amount of the Merger Consideration or provides that the Merger Consideration shall",
-      "id": "87b6c31b04a346b4a3e0da8d2cc5a7ac"
-    },
-    "985344f7ecfb41f4a69ba101973221a1": {
-      "label": false,
-      "text": " During the Employment Period, the Corporation shall pay ----------- the Executive a base salary which, as of the commencement of the Employment Period, shall be at an annual rate of Two Hundred Fifty Thousand Dollars ($250,000). The base salary shall be payable in equal periodic installments which are not less frequent than the periodic installments in effect for salaries of other senior",
-      "id": "985344f7ecfb41f4a69ba101973221a1"
-    },
-    "5d53ff48376046fdab41e95c7f4bad54": {
-      "label": true,
-      "text": "This Agreement may be terminated at any time prior to the Closing Date solely:",
-      "id": "5d53ff48376046fdab41e95c7f4bad54"
-    },
-    "bdeb785be2154b21b4eb052466fa9bcb": {
-      "label": true,
-      "text": "(a) This Agreement may be terminated by you by notice to the Company at any time prior to the Closing Date if any of the following has occurred: (i) since the respective dates as of which information is given in the Registration Statement and the Prospectus, any material adverse change or any development involving a prospective material adverse change in or affecting the earnings, busi ness,",
-      "id": "bdeb785be2154b21b4eb052466fa9bcb"
-    },
-    "fe6871e9070441f8a9e4b3db26b077d7": {
-      "label": true,
-      "text": "Section 3(b), this Section 7 and Section 8 of this Agreement shall survive a termination of this Agreement pursuant to (a) or (b) above in this Section 7 until the date that is two years following the date of such termination. Notwithstanding anything else to the contrary contained herein or in the Merger Agreement, if the Effective Time occurs, the representations and warranties contained in",
-      "id": "fe6871e9070441f8a9e4b3db26b077d7"
-    },
-    "bf1a51751d0748e58c344aec8e5fc789": {
-      "label": false,
-      "text": "This Agreement may be executed in one or more counterparts (including counterparts executed and delivered by facsimile, which shall be as counterparts executed and delivered manually), all of which shall be considered one and the same agreement and shall become effective when one or more counterparts have been signed by each of the parties and delivered to the other party, it being understood that",
-      "id": "bf1a51751d0748e58c344aec8e5fc789"
-    },
-    "bc1b2affa6d848fd92d4dee033e30659": {
-      "label": false,
-      "text": "would, in your judgment, make it impracticable or inadvisable to market the Units or to enforce contracts for the sale of the Units, (iii) suspension of trading in securities generally on the New York Stock Exchange, the American Stock Exchange or the Nasdaq National Market or limitation on prices (other than limitations on hours or numbers of days of trading) for securities on any such Exchange,",
-      "id": "bc1b2affa6d848fd92d4dee033e30659"
-    },
-    "67a73d5887f74a91bed190ca8f64b17c": {
-      "label": false,
-      "text": " The authorized capital stock of FM consists of 1,000 shares of Common Stock, no par value each, of which 1,000 shares are issued and outstanding. There are no outstanding or authorized options, warrants, calls, subscriptions, rights (including any preemptive rights or rights of first refusal), agreements or commitments of any character obligating FM to issue any stock or any other Equity",
-      "id": "67a73d5887f74a91bed190ca8f64b17c"
-    },
-    "025b2ca5147849c8a921d9aaa31cd9cd": {
-      "label": false,
-      "text": "Taxes that are being contested in good faith by appropriate proceedings, provided that Holdings, the Borrower or Restricted Subsidiary, as the case may be, has set aside on its books adequate reserves therefor in accordance with GAAP.",
-      "id": "025b2ca5147849c8a921d9aaa31cd9cd"
-    },
-    "76acff27f13743f4822a094c707d8b75": {
-      "label": false,
-      "text": "have been a suspension or material limitation in trading in the Company\u2019s common stock on the New York Stock Exchange; (iii) there shall have been a general moratorium on commercial banking activities declared by either federal or New York state authorities or a material disruption in commercial banking or securities settlement or clearance services in the United States; (iv) there shall have been",
-      "id": "76acff27f13743f4822a094c707d8b75"
-    },
-    "b11a95c0eb564445b1a473e90622f861": {
-      "label": true,
-      "text": "10.1. This Agreement will terminate:",
-      "id": "b11a95c0eb564445b1a473e90622f861"
-    },
-    "d536428a02084d94ba18d412851cb913": {
-      "label": false,
-      "text": "may not be limited to his Base Salary and that the Employee may receive an annual bonus in the amount, if any, determined annually by the Employer. The Employee shall also participate in employee compensation and benefit plans available generally to executives of the Employer (including, without limitation, any tax-qualified profit sharing plan, nonqualified profit sharing plan, life insurance",
-      "id": "d536428a02084d94ba18d412851cb913"
-    },
-    "368bb1d9c7d0419d9ca58f28565eeb2e": {
-      "label": true,
-      "text": "This Agreement may be terminated in the absolute discretion of the Representatives, by notice to the Bank, if after execution and delivery of this Agreement and prior to the Closing Date (i) there has been, since the date of this Agreement or since the respective dates as of which information is given in the Registration Statement, the Time of Sale Information or the Prospectus, any material",
-      "id": "368bb1d9c7d0419d9ca58f28565eeb2e"
-    },
-    "1b5fd7b037a84404bf85c858953c79e8": {
-      "label": true,
-      "text": "however, (i) the right to terminate this Agreement under this Section 8 shall not be available to such Buyer if the failure of the transactions contemplated by this Agreement to have been consummated by such date is the result of such Buyer\u2019s breach of this Agreement and (ii) the abandonment of the sale and purchase of the Notes and the Warrants shall be applicable only to such Buyer providing",
-      "id": "1b5fd7b037a84404bf85c858953c79e8"
-    },
-    "6d5a23d2663f457cab96df03d9dc8ab7": {
-      "label": true,
-      "text": "In addition, any Stockholder may terminate this Agreement if Weatherford, WEUS, or the Company breaches any representation, warranty, covenant or other agreement contained in the Merger Agreement that (A) would give rise to the failure of Weatherford, WEUS, or the Company to satisfy any condition set forth in Section 8.2(a) thereof, and (B) cannot be or has not been cured within 45 days after the",
-      "id": "6d5a23d2663f457cab96df03d9dc8ab7"
-    },
-    "4a8223a48f83491b9b3eafd7ad37baf9": {
-      "label": true,
-      "text": "The obligations of the Underwriters hereunder may be terminated by the Representatives, in their absolute discretion, by notice given to and received by the Depositor or the Bank prior to delivery of and payment for the Notes if, prior to that time, any of the events described in Section 5(v) shall have occurred or any of the other conditions described in Section 5 shall not be satisfied.",
-      "id": "4a8223a48f83491b9b3eafd7ad37baf9"
-    },
-    "fbb152eae00c440bb2d0df0fbd82c262": {
-      "label": true,
-      "text": "Either of the parties hereto may terminate this Agreement by giving to the other party a notice in writing specifying the date of such termination, which shall be not less than 60 days after the date of receipt of such notice. In the event such notice is given by the Customer, it shall be accompanied by a copy of a resolution of the Board of Directors of the Customer, certified by its Secretary,",
-      "id": "fbb152eae00c440bb2d0df0fbd82c262"
-    },
-    "1d21880f426c45ada31409d22815cc87": {
-      "label": false,
-      "text": "Prospectus or the Final Prospectus (exclusive of any amendment or supplement thereof or thereto after the date hereof).",
-      "id": "1d21880f426c45ada31409d22815cc87"
-    },
-    "795cac72a3504740bc7401a84fc6fba4": {
-      "label": true,
-      "text": "This Agreement may be terminated by the Customer or the Bank by giving ninety (90) days written notice to the other, provided that such notice to the Bank shall specify the names of the persons to whom the Bank shall deliver the Assets in the Accounts. If notice of termination is given by the Bank, the Customer shall, within ninety (90) days following receipt of the notice, deliver to the Bank Instructions specifying the names of the persons to whom the Bank shall deliver the Assets.",
-      "id": "795cac72a3504740bc7401a84fc6fba4"
-    },
-    "3b82e6eba4894ac0b9f7f12aba2aab2e": {
-      "label": false,
-      "text": "of this Agreement, or to Authorized Persons, or may continue to hold the Assets until Instructions are provided to the Bank.",
-      "id": "3b82e6eba4894ac0b9f7f12aba2aab2e"
-    },
-    "da16bd0e9dce4d4c87400eab61b9b14c": {
-      "label": false,
-      "text": "into force of the Convention. In such event, the Convention shall cease to have effect:",
-      "id": "da16bd0e9dce4d4c87400eab61b9b14c"
-    },
-    "02cc328109984db094b0b02caec0d575": {
-      "label": true,
-      "text": "Survival.  The rights and obligations contained in Sections 3 (\u201cOwnership of Work Product\u201d), 4 (\u201cOther Rights\u201d), 5 (\u201cLicense to Preexisting IP\u201d), 6 (\u201cRepresentations and Warranties\u201d), 8 (\u201cConfidential Information\u201d) and 12 (\u201cNon-solicitation\u201d) will survive any termination or expiration of this Agreement. ",
-      "id": "02cc328109984db094b0b02caec0d575"
-    },
-    "f8edf65d9acf4ff4a04459a3492ac426": {
-      "label": false,
-      "text": "Severability.  Should any provisions of this Agreement be held by a court of law to be illegal, invalid or unenforceable, the legality, validity and enforceability of the remaining provisions of this Agreement will not be affected or impaired thereby. ",
-      "id": "f8edf65d9acf4ff4a04459a3492ac426"
-    },
-    "5a8517f359494ead8c11b6aff440480d": {
-      "label": false,
-      "text": "\u0095\tCommitted to deliver the best, we leave no room for customer grievances.\r\n\r\n",
-      "id": "5a8517f359494ead8c11b6aff440480d"
-    },
-    "a47d327d0f6e46fc861f86b2e0e54a2f": {
-      "label": false,
-      "text": "the due diligence and using our agreement creator to close the deal successfully. \r",
-      "id": "a47d327d0f6e46fc861f86b2e0e54a2f"
-    },
-    "811d0dcc92e14c5c881e903c7d4ff7b6": {
-      "label": false,
-      "text": "in accordance with customary procedures in the relevant markets, but in any event for a settlement period no longer than three months following the date of such commitment.",
-      "id": "811d0dcc92e14c5c881e903c7d4ff7b6"
-    },
-    "907f92e0d5704418944a559a4bfb96c7": {
-      "label": false,
-      "text": "terminate in accordance with Section 2 of the Investors\u2019 Rights Agreement.",
-      "id": "907f92e0d5704418944a559a4bfb96c7"
-    }
-  },
-  "version": 33,
-  "description": "Termination or survival clause in a legal document"
-}

data/concept/lilac/negative-sentiment/concept.json

@@ -1,634 +0,0 @@
-{
-  "namespace": "lilac",
-  "concept_name": "negative-sentiment",
-  "type": "text",
-  "data": {
-    "0": {
-      "label": true,
-      "text": "Starting To Be Annoyed By Becky...: I'm not sure why I keep reading these books, but I guess it's because I've read the first two so I'll keep reading the rest of the books. In the first book, I really found it amusing. I was a little annoyed by the fact that Becky couldn't stop spending, but then again that's why she is called a Shopaholic. In the second book, I felt more of the same it was just magniifed more. Now in the third book, I'm just down right annoyed by Becky Bloomwood. In this book, she wasn't going on crazy shopping sprees, just planning two different weddings because she was afraid to tell each person and because I feel she's really selfish. Still, I read the book because I wanted to see how she could get herself out of another situation. I will say that I love her friends Suze and Danny, her client Laurel and her husband Luke. Maybe that's why I keep reading. I will read the next book, but I'm sure I'll be just as annoyed when I'm done.",
-      "id": "0"
-    },
-    "1": {
-      "label": true,
-      "text": "the cover is fine - the pool is horrible: The entire pool was horrible. The cover was fine once we got it on, but we finally got rid of the pool after 2 weeks because it was so hard to set up and keep clean.",
-      "id": "1"
-    },
-    "2": {
-      "label": false,
-      "text": "Good album, not their best.: This album is probably the most marketable and radio friendly of all of dashboard's albums. For the peripheral listener it may be the right one to get to introduce you to this band. But as a Dashboard fan of 5 or so years I truly hope they return to their original sound for their next work. Not for the listen-ability but for the show. To this day the fans react best to the songs from \"Places\" or \"A Mark, A Mission.\" I recommend this album to everyone but I also recommend any and all of their other work.",
-      "id": "2"
-    },
-    "3": {
-      "label": true,
-      "text": "This is a horror novel, right?: Never one to pass up any vampire novel, I purchased Sips because the description seemed interesting. Vampires, Marquis de Sade, fetishism, yada yada yada. If this is a comedy, I give it 4 stars; however, I'll give it 1 star as a horror novel. Sade was rather boring; I would think a character as intense and multi-faceted as the Marquis de Sade would make for a more interesting vampire. The writing style isn't too bad, but overall I found the characters to be mildly amusing at best. The plot was thin, the end was anti-climactic, and the vampires were not very frightening. The book had little suspense, and it leaves a mile-wide opening for a sequel at the conclusion. I would, however, like to see something more of the vampire mutants lurking in the graveyard. They were the most riveting of any of the characters.",
-      "id": "3"
-    },
-    "4": {
-      "label": false,
-      "text": "Superb mix of global non secular musical denominations: I first heard Ms. Pook's music on the \"Eyes Wide Shut\" soundtrack (the masquerade ball scene) and was blown away; if ever there was a necessity for music to permeate a scene in a film this was it. She incorporates a blend of the traditional songs from faiths across continents and mixes them, for lack of a better comparison than similar to your quintessential raver d.j. (though these are better and definitively more original :) \"Oppenheimer\" is my favorite, and if you let the last track run for a few minutes a portion of the song will play once more. I can't wait to hear more of her stuff - these hymns are awesome.",
-      "id": "4"
-    },
-    "5": {
-      "label": false,
-      "text": "A moving and suspenseful story!: For anyone familiar with the occult, this book is likely to raise hairs on the back of your neck as you read. Even if you're not, the storyline is suspenseful and fascinating, and the characters evoke great sympathy and admiration. An excellent read.",
-      "id": "5"
-    },
-    "6": {
-      "label": false,
-      "text": "Simple & Easy to Use - A Practical approach to eating out: This guide is extremely to use. It provides sample menus that you'd see at Chinese, Indian and Thai restaurants. Then you are provided with descriptions of each dish and how it is prepared and the ingredients used. From there you are provided with specific considerations as to how the preparation or ingredient list may affect you if you have Gluten or Allergen issues.This book is the size of a passport and very organized and well written. The Chinese, Indian and Thai Cuisine Passport is perfect for making choices while traveling, or while dining at your favorite local restaurant.",
-      "id": "6"
-    },
-    "7": {
-      "label": true,
-      "text": "Being Fair....I am a fan of hers: and I really enjoyed her previous works, more than I could have ever imagined, but this record is horrible. The songs are trite, the lyrics are incredibly boring, indulgent and immature. The music is pop staple, with forgetable melodies and repetative chorus lines, I feel as if the studio wrote the entire album for her while she was sleeping, this just doesn't speak to any of her previous works at all. This album fits on the same shelf with a Nickelodeon-themed CD. Instead of heading in the direction of an artist like Alanis Morrisette, she is going backward and joining the ranks of Hannah Montana and the Naked Brothers Band. She is a great artist and her first two records are amazing. She is better than this CD and I am looking forward to her next effort.",
-      "id": "7"
-    },
-    "8": {
-      "label": true,
-      "text": "Sucked: I thought the DVD sucked tremendously. It was very boring and if I could, I would return it for a refund. There was only one \"small\" clip of Dylan himself. I'm very disappointed.",
-      "id": "8"
-    },
-    "9": {
-      "label": false,
-      "text": "Excellent product: Easy to install. If you have a newer furnace you probably do not need the swail switch as the HE220A comes with a Humistat which can be connected to the furnace. They recommend the Honeywell 32005847-001 Installation Kit, Bypass which is a little pricey and you can probably buy the pieces of this kit cheaper individually from Home Depot or Lowes or ACO as well as the filters.",
-      "id": "9"
-    },
-    "10": {
-      "label": false,
-      "text": "Very happy.: I am very happy with this trashcan. I was unable to find one in the stores to fit the space in my cabinet, but this one does the job. It is very sturdy and looks like it will put up with years of use.",
-      "id": "10"
-    },
-    "11": {
-      "label": true,
-      "text": "These aren't Throughbreds!: This makes me so mad. All these new authors are coming and changing the series. Nothings the same anymore and the plots are repeditive. Don't even bother reading these books until #32 these are like a different series. I don't know excactly what's happing but these new authors suck!",
-      "id": "11"
-    },
-    "12": {
-      "label": true,
-      "text": "Large and slow are a bad combination.: I bought this TV and returned it a week later, because it blurred so badly with motion that sports were unwatchable. I ended up buying a smaller Sony XBR4, and I have none of the issues (plus the picture is far, far better).This has nothing to do with 60 vs 120Hz. That is more important for DVDs and Blu-Ray signals that are 24fps (which doesn't divide evenly into 60 but does for 120). The LT52133 has an 8ms response time, which is extremely slow. A decent LCD should be 5 or lower.If you want an LCD, choose speed and quality over size. If you want size and quality but want to spend less, buy a plasma. Don't buy a big, cheap, slow LCD!I gave it 2 stars because I like the interface and remote.",
-      "id": "12"
-    },
-    "13": {
-      "label": true,
-      "text": "Skip it: This movie is very poorly written and the result is not distressing, just lame. The actors do their best but from very early on it is obvious that the material gives them nothing to work with. Fans of Colin Firth will experience a certain dim level of enjoyment. Minnie Driver is a treat but her character is no better written than the others. Vermont locations are worth something. With one or two moments of exception it's neither comedic nor romantic.",
-      "id": "13"
-    },
-    "14": {
-      "label": false,
-      "text": "Belive it i went to the concert?: hi everyone let me tell you i went to the concert i was amazed with what i saw cher was awsome i tell you buy the dvd. as i sat in front of the stage cher was doing a great job to us the she is living proof . So i urge you to buy it?",
-      "id": "14"
-    },
-    "15": {
-      "label": false,
-      "text": "Vale la pena.: En este libro se narra de una forma muy interesante la vida de una familia en particular. Lo que mas me gusto de este libro fue la manera en que la autora describe a lo largo del libro las personalidades de los sujetos envueltos en la novela; que vienen a ser muy distintos y extremos, lo cual, intensifica el drama... Definitivamente es un buen libro y lo recomiendo a todos.",
-      "id": "15"
-    },
-    "16": {
-      "label": false,
-      "text": "Nummie Children's story: I ordered this book for our grandchildren. Two boys 5 & 3 and a 4 month old girl. All love the story. The mouse is determined.",
-      "id": "16"
-    },
-    "17": {
-      "label": true,
-      "text": "Seem to be alone on this one: Looking at the other reviews, I seem to be the only one that was disappointed with this book. The content is too babyish in most of it for older tweens and the more \"grown up\" content would be over a younger tween's head. I had a quick read through and with every paged turned, I thought duh. I'll be looking around for another book shortly.",
-      "id": "17"
-    },
-    "18": {
-      "label": false,
-      "text": "Best yet: by far the best EA game yet. I especially like the easy controls and kick - a graphics. the playbook is extremely accurate and detailed. Also the fight songs and cheerleaders were a nice touch. this is an excellent game and worth checking out.",
-      "id": "18"
-    },
-    "19": {
-      "label": true,
-      "text": "washed out: A bit like Simply Reds version of the Valentine bros hit \"Moneys too tight to mention\" - this cover version has nothing of the driving energy that characterised the original recording.",
-      "id": "19"
-    },
-    "20": {
-      "label": false,
-      "text": "great water bottle: I love this bottle it is great. I like knowing it is non toxic and it just works very well. You can have it full and lay it down and it doesn't leak at all.",
-      "id": "20"
-    },
-    "21": {
-      "label": false,
-      "text": "Nice goggles: I am pretty happy with these goggles. They work well during swim workouts in the pool. I do notice a little bit of fogging from time to time. I had hoped to wear them during an upcoming triathlon, but based on a few instances where they slipped a little in the pool I am concerned that they won't be secure enough. I will keep using them in the pool, but will likely get different ones for open water races.",
-      "id": "21"
-    },
-    "22": {
-      "label": true,
-      "text": "aaahhh nnnoooooo!: Hopefully the last film in one of the worst horror trilogys ever made. This series pretty much ruined the horror film for years to come, for one its too self aware, thats incredibley annoying, second everyone acts like they are on Friends or some sitcom. The acting is just plain bad and unconvincing. Now the gore, if you're going with material this weak you should load it up with disgusting violence, is there any in the Scream series? No.Everyone went to see this movie just to see who THE KILLER is. This movie sets low standards to be met, you expect alot of people to die, one shock, then we find out who the killer is, then you go home. Every horror film being made today is like that, there's nothing new or exciting or risk taking, its the same stuff over and over and people are laping it up like dog food.This film is what you've come to expect, bad acting, some people die and we eventually find out who the killer is and all is merry and well. Pathetic.",
-      "id": "22"
-    },
-    "23": {
-      "label": false,
-      "text": "A classic of its kind: This movie is a classic of its kind and much better that a lot of movies, that followed. It is not one of the best, but it still deserves five stars...",
-      "id": "23"
-    },
-    "24": {
-      "label": true,
-      "text": "Nice suite, but Virtual PC 7 disappoints on my G5: I purchased the upgrade since I'd already bought both Office v.X and Virtual PC 6.1 last year.The biggest letdown is that Microsoft's promised support for the G5 is nearly non-existent. I have a dual processor G5 with an ATI Radeon 9800 card (Apple), and after trying to install Virtual PC 7 three times, I cannot get a VM to work. It did install (and work) flawlessly on my G4 Powerbook. Googling for reviews finds it's very hit or miss, but if (when) it misses, you'll regret investing the extra $$$ in an immature product.",
-      "id": "24"
-    },
-    "25": {
-      "label": true,
-      "text": "Okay player, don't expect a miracle: I bought this DVD player at Circuit City earlier this yr for about a $100. I hooked it up to a 47\" Vizio LCD (which by the way has an awesome picture) using a HDMI cable. After fine tuning this product, I was very, very, very diasppointed. The picture was very \"grainy\" (lots of pixels). I have a $35 DVD player that only utilizes an s-video cable that produces a much more crisp picture. Be warned, the picture stinks.",
-      "id": "25"
-    },
-    "26": {
-      "label": false,
-      "text": "A revelation of the science of consciousness evolution and all natural growth: Here is a readable and fascinating account of the development of the new science of chaos theory, the only body of ideas that describes how the natural world as experienced by human beings emerges out of basic quantum processes. The different explorers and innovators of the new science are introduced in a personable way that will enchant the interested reader.",
-      "id": "26"
-    },
-    "27": {
-      "label": true,
-      "text": "Don't say that I didn't warn ya' !: I'm absolutely convinced that Delbert McClinton had no controlover the release of this CD. I rated it 1 star simplybecause there is no 0 star rating ! In actuality , I am not certain that the vocalist on this recording IS Delbert McClinton. Only on the Mr. Pitiful track is there any similarity at all to Delbert's voice. This is the perfect CD for someone with money to burn who would like to have a recording of a 1960's garage band recorded in a garage and who should be working in a garage ! Delbert fans...run fast and run far away from this ! END",
-      "id": "27"
-    },
-    "28": {
-      "label": true,
-      "text": "This item is not available: I ordered this unit on February 7th. Every time I checked back on the status of the order, it read \"not shipped\" and the estimated shipping date got moved out. I really don't think this unit is avaialble from the company anytime soon. I cancelled the order.",
-      "id": "28"
-    },
-    "29": {
-      "label": true,
-      "text": "I used to like ABBA...: I used to like ABBA, until I saw Mama Mia! A horribly disjointed musical, where songs feel contrived to fit into the story; a story that doesn't seem to come together. Individual songs are usually done alright, but don't segue from one to another very well.The cast butchered several of the songs, but especially S.O.S, Take A Chance On Me, and anything where Pierce Brosnan sang. On a side note, I also counted at least two violations of Chekov's Gun. And finally, I think it has a bad moral message. Which you only recognize if you manage to sit through the whole thing.If there is justice in the world, cast members without established careers won't get to have them as punishment for the worst movies I've seen since The Talented Mr. Ripley.",
-      "id": "29"
-    },
-    "30": {
-      "label": true,
-      "text": "A complete disaster!: If you're like me, you probably wanted to check out this movie because it sounded like it really could be an excellent supernatural Gothic horror tale full of goblins and wicked things alike. Well, don't make the same mistake I did and actually watch it. It's horrible. Terrible. An honest to goodness waste of film. The acting is wretched, the film quality is rotten (it actually looks twenty years older than it is), and the plot is thin, weak, and does not give you what it's supposed to. The only reason I bothered to give this film 1 star is because of Alexis Arquette -- he's great looking, but should have left this film out of his career.",
-      "id": "30"
-    },
-    "31": {
-      "label": false,
-      "text": "beautiful detail: I just purchased these Dover COloring Books for my mother and she loves them. The detail is out of this world and the variety of colors you can use are only limited by your inagination. HIGHLY RECOMMENDED!",
-      "id": "31"
-    },
-    "32": {
-      "label": true,
-      "text": "Very disappointed: I looked forward to getting this movie as I had heard many good things about it but it was nothing like I had imagined or been led to believe. There is very little actual history in it or real Christian experience except for the background because the main focus is a soap opera style romance and caricature figures. I agree with the reviewer who described it as a mixture of \"tawdry Hollywood sex\" somehow interspersed with a vague nod to Christianity. The only decent scene was the arena scene where the Christians are going to their deaths singing hymns - but that's not enough to make it a great or even a good movie. Not personally to my taste anyway.",
-      "id": "32"
-    },
-    "33": {
-      "label": true,
-      "text": "Unreliable minikit: I bought this minikit because it got good reviews and it would be perfect for my purposes. However it switches on and off whenever it wants, it looses contact with the phone. Very often the on/off button works only in a horizontal position (?) I use a Treo 650, which is on the compatible phone list. When I contacted Parrot, they said it wasn't (?) At last I opened the unit, but there are no moving parts inside except the micro switches. It is giving me a headache, so I will go searching for an alternative.",
-      "id": "33"
-    },
-    "34": {
-      "label": false,
-      "text": "A Christmas Classic!: This is surely one of the best classical Christmas recordings available. Don't buy the older version, as the quality of this recording is excellent. This is one of those \"Every Christmas - Can't have Christmas without\" recordings.",
-      "id": "34"
-    },
-    "35": {
-      "label": true,
-      "text": "too narrow: These were the narrowest pair of D size shoes I have ever tried on. I don't care how nice a shoe looks. If it don't fit it just don't fit.",
-      "id": "35"
-    },
-    "36": {
-      "label": true,
-      "text": "Lack of extension: This earphones lack a descent extension cord. ITs very small cable, but its of good quality. Sadly, cord its too short, and the extension is useless.",
-      "id": "36"
-    },
-    "37": {
-      "label": false,
-      "text": "Easy-Reading: This is the 3rd Southern Sisters Mystery I've read. They're easy, fast and funny murder mysteries, with lots of cute family stories intertwined in the intrigue.",
-      "id": "37"
-    },
-    "38": {
-      "label": true,
-      "text": "it'd be great if it worked like it was supposed to: for the first 30 seconds it was lovely, but i believe that either the motor isn't powerful enough to keep the shaft rotating smoothly or 3 AA batteries just don't provide enough juice for the motor to work more than 30 seconds. it was a nice idea, but i'm rather dissapointed. the jelly material is somewhat difficult to maintain also. i think if it were hooked up to a larger battery pack it'd be WONDERFUL... which i think i may have a macgyver friend with a knack for electronics attempt to do for me.",
-      "id": "38"
-    },
-    "39": {
-      "label": false,
-      "text": "Not Hornby's best but still good: I loved About a Boy and really, really loved the sardonic wit of High Fidelity. About a Boy is much deeper but just as cynical. Maybe even more so. The characters are richly drawn and just complex enough to keep the reader wanting more. Good read, but best to take some time with this one. Not recommended for a summer beach read.",
-      "id": "39"
-    },
-    "40": {
-      "label": true,
-      "text": "A Disappointment: As with most Taunton Press publications, the illustrations and photographs in this book are spectacular and the organization and layout is superb. Nonetheless, I found this book disappointing. It lacks both depth and breadth. I had hoped for a detailed review of wood joinery including some of the more unusual joinery found in Japanese woodworking. This book, however, is targeted more toward the beginner. Even so, it does not cover the details and \"tricks\" of even the most basic techniques in sufficient detail to allow beginners to easily reproduce them. Consequently, it is unclear who this book was written for - not the beginner as it lacks depth, and not the advanced woodworker as it lacks breadth. Far more effort appears to have been put into appearance and organization than in content.",
-      "id": "40"
-    },
-    "41": {
-      "label": true,
-      "text": "Horrible. Don't do it!: Great price for the item when a 6' one of these at Best Buy is $20. Thing is, the one from Best Buy fits in the outlet and stays there. This cord fits very loose and does not connect. I bought 2 of them, neither did what they were suppose to.As much as I hate to say it, but, buy the more expensive one. At least it works.",
-      "id": "41"
-    },
-    "42": {
-      "label": false,
-      "text": "Given as a gift...: Given to my best friend as a gift. She loves it. Her fiance enjoys making coffee for her in the mornings. :)",
-      "id": "42"
-    },
-    "43": {
-      "label": false,
-      "text": "Love the ring.: This is a nice ring. I was worried it out be thin and cheap looking, but it's not. It's a very pretty stylish ring. Go for it.",
-      "id": "43"
-    },
-    "44": {
-      "label": true,
-      "text": "Beautiful writing Marred by One-Note Characterizations: How could Kingsolver have ruined her book with such an obvious error? Nathan is a strident paper doll that flattens the whole story. Just as bad, the author has all the narrators using the same ironic tone to decribe him, deadening their voices as well. At the same time, Kingsolver doesn't have the guts to show him doing something trully terrible. I don't trust an author who can't let the reader make up his own mind, and as a consequence I couldn't trust her views about ANYTHING in the story. I'm giving this two stars for her descriptions of the African landscape, and that is all.",
-      "id": "44"
-    },
-    "45": {
-      "label": true,
-      "text": "Much worse than any cordless phone I've ever had: This phone cuts out only 2 rooms away from the base station. There is static noise, and callers on the other end complain about sound quality. I can't go into the garden, which used to be no problem with my old 900 MHz phone.",
-      "id": "45"
-    },
-    "46": {
-      "label": true,
-      "text": "Waste of time & money: The first Hangover was not too bad, this one was just terrible. The acting is bad, the script is bad, everything about this movie was just bad. Do yourself a favor, don't buy this movie as it is a total waste of time and money.",
-      "id": "46"
-    },
-    "47": {
-      "label": true,
-      "text": "Did Not Work For Me!: Impressive You Tube Video (Like a Sci-Fi Fantasy). In reality it's a high speed Easy Out so unsurprisingly it broke faster than an Easy out. This product did not work for me. The drill part did not drlil, the puller part did not pull. It was a total zero.",
-      "id": "47"
-    },
-    "48": {
-      "label": false,
-      "text": "Excellent book, long overdue.: From a very long time women were told that looking good was of utmost importance. This was without regard to health or fitness and how age affected these parameters. Witness the whalebone and other types of corsets, the spike heeled shoes and the numerous weight loss programmes on the market (some of which are downright dangerous). Now there is a book, backed by solid research, that allows women of all ages to remain fit and healthy for a lifetime. I am certainly going to recommend this book to all the women I know.Bentley Norville",
-      "id": "48"
-    },
-    "49": {
-      "label": true,
-      "text": "not an all star: Not a practical guide in this collecting age. Does NOT have a comprehensive list; meaning it does NOT cover all manufacturers and, more importantly, for the ones it does, only provides listings of the base set. That means no insert or variation pricing whatsoever. Also, no oddball or minor league issues are listed. Generally speaking, unless you are collecting base sets prior to the advent of inserts and alternate versions of the base set, this guide is fairly useless.",
-      "id": "49"
-    },
-    "50": {
-      "label": true,
-      "text": "Again, second rate city, third rate writer: Just another example of Mr. Lindberg's pitiful attempt at exhibiting a strong expertise on a subject with which he is clearly obsessed. Don't waste your time with this book, either. It is poorly written and fails to engage the reader. You might consider using this book and the first book he wrote on the same subject, as a pair of bookends. That is about all they are worth.",
-      "id": "50"
-    },
-    "51": {
-      "label": false,
-      "text": "Reality: No one should need to convince you to buy this book, you should just do it! It's so well written and worded and brings you right to the heart of a sexual reality that most people like to pretend doesn't really live and breath in their fair cities. I never again want to hear someone bad mouth a working girl for what she does. I will and do now however look at men with a curious eye wondering if they are depraved peep show window lickers :)",
-      "id": "51"
-    },
-    "52": {
-      "label": true,
-      "text": "Bummer: Visual effects and Battle footage were great...the other 85% of the movie was just lousy fluff...",
-      "id": "52"
-    },
-    "53": {
-      "label": false,
-      "text": "The spark of idependence: Filled with the independent spark that made us all love life at one point or another. A fun, introspective and nonsensical movie that sticks with you.",
-      "id": "53"
-    },
-    "54": {
-      "label": false,
-      "text": "What I expected from Mirman's website. Funny. Funny. Russian.: lol, gotta love Eugene. Even when his audience doesn't initially laugh, he gets in a good zinger at himself and they laugh at that. He's witty without being condescending, and uncomplicated without seeing contrived. However, if you're not a fan of irreverant humor, this may not be for you.",
-      "id": "54"
-    },
-    "55": {
-      "label": true,
-      "text": "Do not...repeat...do not bother!: It is not often that I offer a negative review but this compilation while attractive does not deliver at all.The foot massage gizmo is awkward and uncomfortable.The pumice stone leaves rough splinter like skin.The foot scrub doesn't reall scrub.The rotary action tool has five heads, none of which work well and you must hold the switch in place or it turns off. It is cumbersome and ineffective.The one star was initially given for a foot brush (which later lost its bristles very easily as I update the review) and a sweet smelling foot repair balm.Don't waist your money. Soak your feet and invest in an inexpensive German Titania file, smooth and coarser side, or a like product. It will last for years.",
-      "id": "55"
-    },
-    "56": {
-      "label": true,
-      "text": "Not Sandra's Best: Ms. Brown has written better romance novels. Don't give up on her if this was your first Sandra book.The feeble female lead struggles with a 15-year crush that walks back into her life. The smug male lead acts like a jerk through most of the novel. The romance scenes grapple to muster up passion but fall short. Both of the main characters bothered me; my favorite character was the 17-year old.A quick read...about 4 hours (with interruptions) for me...but probably not worth it.",
-      "id": "56"
-    },
-    "57": {
-      "label": false,
-      "text": "Impressed: Lots-O-Fun. Wood and glass toys are high quality and are a good fall back for the kids to play with they are \"bored\". Would buy again.",
-      "id": "57"
-    },
-    "58": {
-      "label": true,
-      "text": "Light turned on by itself 3 times: The installation was easy. I used it for a week, everything worked fine, EXCEPT the light it connected to turned on by itself 3 times so far, with no one near to either one of the switch. Not sure whether it is a defective unit, or this product is too sensitive to noise. I'm returning this product and will just install a regular switch instead.",
-      "id": "58"
-    },
-    "59": {
-      "label": false,
-      "text": "good battery: I feel kind of silly writing a review for a battery, but have to say that these last a LONG time. Work very well.",
-      "id": "59"
-    },
-    "60": {
-      "label": false,
-      "text": "Even a Woman finds it funny: Yes, even a woman finds \"Married to Mommy\" funny. The book gets you laughing aloud when it is trying to make fun of \"Mommies\". The truth is that it really is making fun of the stupidity of men and their simple basic needs of sex, getting out of work, and beer. Of course, the truth is always funny.A definite MUST for any woman, married or not. We will now know all the secret tricks the men try to use on us.By the way, I am NOT a MOMMY!",
-      "id": "60"
-    },
-    "61": {
-      "label": false,
-      "text": "Gungrave...not quite what you might expect: Those thinking this is another version of Trigun will be disappointed. Gungrave is actually a lot deeper and more complex. The lead is short on dialouge, but the story has more depth and character development than most anime. The first DVD is more about the main character's past than about the reanimated killing machine he's become, but it definitely leaves you wanting more.",
-      "id": "61"
-    },
-    "62": {
-      "label": false,
-      "text": "Error in product description: It's great in every way. However, if you'd prefer a digital tuner (as I do), then you might need to look further. The product description boasts a digital AM/FM tuner, but it's disappointingly an analog AM/FM tuner.Overall - especially for the price - I think it's pretty good.",
-      "id": "62"
-    },
-    "63": {
-      "label": false,
-      "text": "good phone but not as user friendly as it could be: Battery life is very good. Phone has good range. My only complaint is it's to involved to get your message from the handset.",
-      "id": "63"
-    },
-    "64": {
-      "label": true,
-      "text": "Big waste of money (and space in my house!): My 5 year old son wanted this so bad, but when we got it for him, there were so many pieces to put together that didn't fit together well, he never played with it. It just sits on our floor in many pieces taking up toy space! What a waste!",
-      "id": "64"
-    },
-    "65": {
-      "label": false,
-      "text": "Don't want to take it off: Very satisfied with an earlier purchase of this Bali bra model, I was just as pleased with the new one. Very comfortable, well made and a good neutral color. It will be my next choice, too.",
-      "id": "65"
-    },
-    "66": {
-      "label": false,
-      "text": "Fantastico: If anybody who's into rock music is ever looking for a band to keep you on your toes, this is the band. I've been a fan for 10 years now, and no album has ever sounded like any of their previous albums. This disc is fantastic with such a variety of styles, as are the previous releases, even back to the Rainbow Butt Monkey days.",
-      "id": "66"
-    },
-    "67": {
-      "label": true,
-      "text": "too much visual: There are far too much designs, visuals, colors, etc in the book - this is highly distracting, as TV screen can be...By way of example (among so many...), what is the use of colors with the three squares of the Pyth. theorem???? this is as useless as writting 2+3=5 with 2 in blue, 3 in red and 5 in yellow...I wish I had purchased the 2nd edition, which according to reviews was closer to what I was looking for.",
-      "id": "67"
-    },
-    "68": {
-      "label": false,
-      "text": "Aretha's First Arista Release Showed Pleasures to Come: After a long and musically satisfying career with Atlantic, Aretha severed her ties with that company and moved under the wing of Arista's Clive Davis. With the start of the 1980's, Aretha was looking for new territory to conquer and almost succeeded with this mixed bag.\"United Together\" is a fine tune that benefits from beautiful orchestral arrangement that is matched by Aretha's superb vocal instrument. The remake of \"Can't Turn You Loose\" allows Aretha to show why she is the Queen of Soul\" for she really belts this one out. Another cover, that of the Doobies' \"What a Fool Believes,\" is an interesting interpretation. The final cut \"School Days\" appears to be \"autobiographical\" for every girl growing up in the fifties.Although not as strong as her Atlantic work, \"Aretha\" is still a suitable addition to the artist's discography.",
-      "id": "68"
-    },
-    "69": {
-      "label": true,
-      "text": "Misguided Purchase: The photo and description do not reflect the product. The screen panel kit I received was white. What a huge inconvenience during a time-crunch.",
-      "id": "69"
-    },
-    "70": {
-      "label": true,
-      "text": "Banacek: My husband and were looking forward to seeing this series.The first show was SO boring, we finally just quit watching it.Actually, we haven't gotten around to watching anymore. I guess we were afraid of a repeat.Maybe that was just once, I hope!",
-      "id": "70"
-    },
-    "71": {
-      "label": false,
-      "text": "JDT: Uncle Tupelo is without doubt one of the most under appreciated groups of the 90's. Anodyne, like each of the three albums that came before it, has everything that a remarkable recording requires: great songs, honest lyrics, and artists who really care about the music they are making. Like the best of Dylan and Springsteen, the songs are about real people with real troubles and joys. When you hear them you know they are coming from the heart. The songs contributed by Jay Farrar and Jeff Tweedy are easily differentiated by the voacls, music, and lyrics. What makes this record interesting is how well these unique sounds compliment each other. The union is seamless.",
-      "id": "71"
-    },
-    "72": {
-      "label": false,
-      "text": "Well Worth Reading: First a confession: Miriam Wasserman was my mother. However, she published several books, but this is the only one I really found useful. She walks the reader through the New York City school system and the attitudes of different groups involved in the system back in the 1960s. This includes parents, teachers and administrators. Her view is that the further away one got from parents and students, the more prestige one had. She meticulously describes the teachers' strike of 1968 against \"community control of schools\", a strike of which she is extremely critical. She explores the racism that was involved in this strike, including using quotes from striking teachers, etc. It should be emphasized that the author was pro-union all her life, so her views don't stem from an anti-union bias. The book also covers the high school student rebellion which coincided with and followed the strike.",
-      "id": "72"
-    },
-    "73": {
-      "label": false,
-      "text": "compact and loaded: I bought this phone after reading the cnet reviews and really liked it. It looks small and really compact. I like the camera pics at 2 mega pixel and bright flash. The mp3 player is crisp. The headset that comes along delvers amazing fM radio. I think my phone is not very loud and you have a problem when you are around a noisy crowd. I just bought this phone again for my cousin. He likes it too. Almost forgot the display is very good.",
-      "id": "73"
-    },
-    "74": {
-      "label": false,
-      "text": "Outstanding text!: Brooks/Cole should keep this text in their catalog for ages! It is well-written, examples are generally quite clear, vocabulary is introduced well, and the exercises develop real skills, rather than simply be busy-work. One of the best calculus books ever!",
-      "id": "74"
-    },
-    "75": {
-      "label": false,
-      "text": "Excel 2003 Bible: Very good source of information. I will most likely buy other books in this series.",
-      "id": "75"
-    },
-    "76": {
-      "label": false,
-      "text": "Tasting is Believing: Gluten-free breads used to have a gritty texture from the rice flour, and were too soft for sandwiches. Bette Hagman uses garbanzo/fava bean flour, sorghum flour, tapioca flour, and corn starch to create breads which have a similar texture to wheat flour breads, and the flavors of her breads are fabulous.My BF bought me this book and a great tasting beverage to drink it with. Since he knows I quit coffee recently, he's been really wonderful helping me in cope with my mood swings. S o y f e e is made from soy beans that is roasted just like coffee. I enjoy the taste and don't miss coffee one bit. Buy it online at www.s o y c o f fee.com.This is a 'must have' for anyone baking gluten-free. I think all of Bette Hagman's books are wonderful and a must for those with gluten intolerance.",
-      "id": "76"
-    },
-    "77": {
-      "label": false,
-      "text": "5 stars for the show, no stars for the \"Collector's Edition\": I was really looking forward to getting this Collector's Edition and see what extras were added. I knew it wasn't a lot - just a mini-book and a documentary - but I figured it would be packaged in a cool way.Wrong.As others have already mentioned, the Collector's Edition is *literally* theAvatar: The Last Airbender - The Complete Book 1 Collectionslipped into another cardboard box, with a little booklet and DVD in an envelope (not even a case!) wedged in. It's really disappointing; it would have been so easy to create a quality Collector's Edition but the studio couldn't be bothered, I guess.",
-      "id": "77"
-    },
-    "78": {
-      "label": false,
-      "text": "sula scottcampos: Sula, a book that talks about the issues of being a black women is a really good novel to read.One of the reasons I recommend it is because of its realism and its themes - death, sex, friendship and poverty.I also think that its characters are very good, its easy to identify with one or both of them. I really recommend this book to anyone who enjoys good literature.",
-      "id": "78"
-    },
-    "79": {
-      "label": false,
-      "text": "Fantastic! It's a must-have for girls!: I hated razor, tried shaving but it did not work for me. Shaving made the hair grows thicker and faster afterwards, plus the roots are impossible to be getting rid of. After reading the reviews, I ordered it to try, I used it for once and already fall in love with this. I used to use small tweezer to pluck out my leg's hair, in order to avoid the razor, it took me a few hours to do that but this super electronic tweezer works wonder! You won't see the black roots and I have smooth and silkly legs in 20 mins. It does not hurt at all, if you use it on your legs. But, if you use it at your under arm, it won't be a pleasant feeling, of course! I will never use anything else besides this for hair removing anymore! highly recommended!",
-      "id": "79"
-    },
-    "80": {
-      "label": true,
-      "text": "This is not a toy: I guess I was expecting more out of these leave window decals. I just didn't find them attractive after placing them on my window, they seem very cheap, I guess because they are cheap.I threw them away.",
-      "id": "80"
-    },
-    "81": {
-      "label": false,
-      "text": "Wonderful book for anyone running a professional hatchery: This book is aimed more for hatcheries that are raising Trout, Salmon, Catfish and other food fishes. However, there is so much information in this book that even ornamental fish hatcheries will find an incredible amount of useful information. The chapters on Fish Nutrition are especially helpful.",
-      "id": "81"
-    },
-    "82": {
-      "label": false,
-      "text": "Amazing book!!: Once again, Eric Victorino's artistic talent is put into this great free-verse poetry book. I couldn't put it down and I finished it the day I received it in the mail. All of the poems are awesome but the one I found the most interesting was \"It's A People Business.\" All of the experiences in his life, personally and with his band, come to life in this book. Please check it out! It's worth every penny!!",
-      "id": "82"
-    },
-    "83": {
-      "label": false,
-      "text": "The white trumpet contender respect Miles Davis!: The story of the Jazz in the Fifties certainly would be remain unfinished without the ominous presence of this outstanding virtuoso. Baker sound still possesses this alluring hook, this magnetic engagement charm, eloquent expressiveness, enrapturing lyricism and contagious rhythm, despite the elapsed time, which confirms by itself the status of his musicianship.This selection is jus a little sample of the broad universe of his genius. A well thought selection of great musical successes, available, preserved and immortalized by the Digital Technology for our future enjoyment.Absolutely indispensable in your treasured collection.",
-      "id": "83"
-    },
-    "84": {
-      "label": true,
-      "text": "What the?: I'm sorry, maybe it's just me but I can't helping stating that this has to be one of the wrost movies I've seen in my life!Can you say boring? Can you say doesn't make sense at all? The first 30 minutes of the movie were O.K. But it went downhill after that. This movie is a prime example of a director attempting to make a deep movie with a meaningful lesson but failed on all levels. I don't recommend this movie unless you want to go to sleep or you don't have anything else to do.",
-      "id": "84"
-    },
-    "85": {
-      "label": false,
-      "text": "very very good!!!!: linda blair is a young girl who is possessed. and her mother doesn't know what to do until one day when she hears her daughter screaming and stabbind herself she knows what to do GET AN EXORCIZIM!!!",
-      "id": "85"
-    },
-    "86": {
-      "label": false,
-      "text": "Awesome product for the price!: This range extender works as advertised! I am very happy with the purchase. I was a little worried after reading some of the horror stories here, but I have to say, Chovy's review instructions (on this site) were just this ticket to get the repeater up and running in less than 30 minutes. It was unbelievably easy to install! Do not be frightened by negative reviews. If you can set up a wireless network, you can set up this repeater. However, I did upgrade the firmware before I did anything else and maybe that helped. I got the firmware update from the Belkin site.",
-      "id": "86"
-    },
-    "87": {
-      "label": true,
-      "text": "Slight: This book is either a heavily illustrated short story collection or a text-heavy comic. Its unusual format is its most original feature. Its plots are negligible, but its illustrations and text evoke a unique atmosphere of self-conscious nonconformism. Although its target audience is dare-to-be-different teens and college students, its interesting turns of phrase and expressive line drawings are not devoid of interest for general audences.",
-      "id": "87"
-    },
-    "88": {
-      "label": false,
-      "text": "ANgeleyes: Seem to dry up their eyes fairly well, although I haven't seen the color (brown stain) change much yet.",
-      "id": "88"
-    },
-    "89": {
-      "label": true,
-      "text": "Nice Try: Salt Lake 2002 is not a bad game, but it isn't good either. The graphics are excellent, but some of the events are bad. Bobsleigh, and skiing aren't bad but the others are. You dont stay into it for long. I liked it for a while, but it gets boring.",
-      "id": "89"
-    },
-    "90": {
-      "label": true,
-      "text": "Cutler's share of the pie: This book was a major disappointment. I am familiar with books written solely by the Dalai Lama, such as the \"Library of Tibet\" series, which are much more engrossing and have much more substance than Cutler's book. Cutler attempts (successfully, sadly) to have his share of the profitable market that involves the Dalai Lama's writings. The book is insipid, does not try to explain any important issue in the light of Buddhist philosophy, and only rehashes issues that several other westerners already wrote about. It's another big ego trip: we keep hearing time and again about his opportunities to be with the Dalai Lama. What a shame, Cutler. I sold the book as soon as I finished it.",
-      "id": "90"
-    },
-    "91": {
-      "label": true,
-      "text": "Mostly tedious, with interesting parts: I found the writing interesting, and the subject fascinating, but I found myself frustrated by the author's difficulty in talking directly about the status of Muslim women with her interview subjects. The author spent many pages writing about the menus and dress of the many middle and upper-middle class women she interviewed. It seemed as though her interview subjects resisted her efforts to discuss the status of women in their countries, so we too as readers had to wade through much distracting material and misunderstandings about feminism and gender. Great travel stories, but not a great source of information about Muslim women.",
-      "id": "91"
-    },
-    "92": {
-      "label": true,
-      "text": "Sesame Street Toddler: I did not find this game to be as educationally sound as I would expect from Sesame street. There is too much talking before the program will react to a command. The graphics are jerky and the cursor acts like the target is magnetically charged and keeps pushing away the cursor. When the child actually does manage to click on a target, the cursor may still fly to another target and the child is told that his answer is wrong. Another example of educational problems is the pronunciation of \"eggs\" using a long \"a\" sound instead of a short \"e.\" This is not very helpful in teaching a child the sound for short \"e.\" Children that are used to playing computer games by themselves may find that this game is too frustrating to do alone. The open ended learning curve is a great idea. I just wish Sesame Street would hire a truly qualified literacy expert to help clean up the many problems in this program.",
-      "id": "92"
-    },
-    "93": {
-      "label": true,
-      "text": "needs a buzz cut and a point: I avoided reading this book, not because of the hermaphrodite subject matter, but because I have never read a multigenerational family saga that I liked. Many books let me down in the middle, and this was no exception. The beginning of the book was incredible and harrowing, with momentum and characterization. The post-America nextgens part of the saga was so boring I found myself flipping and flipping - always a bad sign. If there was some kind of larger point to all of that, then I must have missed it. Yes there's the identity duality and trinity themes playing out here: man/woman, greek/turkish/american modern/old world sick/healthy innocent/guilty original/reinvented. But it was almost as if the author was saying - here it is again - get it? I like my fiction much more subtle than this.",
-      "id": "93"
-    },
-    "94": {
-      "label": true,
-      "text": "OMG! DO NOT BUY!: I normally don't take the time to submit a review.In this case however, I feel obligated to do so.This is by far one of the worst purchases I have ever made.Here's why.....The contraption is far too bulky.The case's enclosing is unbearable, takes a good minute or so to open it.The texture of the material feels like a cheap toy.The overall design is horrible, something I could make in my basement.For the love of everything sacred, do not buy this thing.",
-      "id": "94"
-    },
-    "95": {
-      "label": false,
-      "text": "Good price, good quality: Comparable HDMI cables can be bought for 45 or more. Even though the price is cheap the quality is good, no problems so far.",
-      "id": "95"
-    },
-    "96": {
-      "label": false,
-      "text": "Good rock music: This is what i call rock music good beat and good lyrics, don't listen to the other reviews. This cd is one of the best, listen to a few songs and you will get hooked. I recommend this cd its awesome.",
-      "id": "96"
-    },
-    "97": {
-      "label": true,
-      "text": "BORING!: This movie is soo boring. How in the hell did this movie make so much at the box office. Do people really want to pay for crappy movies like this. bottom line this is a chick flick nothing is good. And now they are re-releasing this movie with more boring stuff. This is the worst movie ever.",
-      "id": "97"
-    },
-    "98": {
-      "label": true,
-      "text": "Already Rusting: Inferior quality. The plating is thin and rust is coming through the finish. Inexcusable for a product that is designed for use in a humid environment.",
-      "id": "98"
-    },
-    "99": {
-      "label": true,
-      "text": "confusing internet setup: i wanted a camera that could email photos but this camera will not go out through the router and the manual setup , to punch a hole thru router is confusing.",
-      "id": "99"
-    },
-    "04c7dfc0f94e4e88968d09b40edbfa14": {
-      "label": true,
-      "text": "The new gaming console is unaffordable.",
-      "id": "04c7dfc0f94e4e88968d09b40edbfa14"
-    },
-    "58f58a1a4cbb4bb699772ed934006ec8": {
-      "label": true,
-      "text": "How can it be sure difficult for @115830 to deliver a package to a University address? Two failed attempts so far ...",
-      "id": "58f58a1a4cbb4bb699772ed934006ec8"
-    },
-    "d4a3cd4877c54aef81c376eff8008df4": {
-      "label": false,
-      "text": "@204780 Glad they showed up! Hope you have a great flight! -Sean",
-      "id": "d4a3cd4877c54aef81c376eff8008df4"
-    },
-    "affe1d6548f84bed84238bac45cc10a1": {
-      "label": false,
-      "text": "@British_Airways Thank you! All looks good then \ud83c\uddec\ud83c\udde7\u2708\ufe0f",
-      "id": "affe1d6548f84bed84238bac45cc10a1"
-    },
-    "e304ea77a94c450a95690c7b605a035f": {
-      "label": false,
-      "text": "@246667 Thank you for reaching out, Andrea. The built in application in Windows 10 are exempted to be uninstalled. However, you can send this suggestion directly to our developers via the Feedback Hub so they can take a look at it: https://t.co/jowrfbgQm6. Keep in touch.",
-      "id": "e304ea77a94c450a95690c7b605a035f"
-    },
-    "76b694b019eb4e6888a422e144030bd0": {
-      "label": true,
-      "text": "@GWRHelp It\u2019s mainly the constant short forming and cancellations due to mechanical faults Phil. As a company, these excuses have been used ad nauseam for years and years. It just gets worse and no amount of rhetoric and IET self promotion can hide that fact.",
-      "id": "76b694b019eb4e6888a422e144030bd0"
-    },
-    "ce0698020b7a457396c7674b04db10e6": {
-      "label": false,
-      "text": "English gangster flick.",
-      "id": "ce0698020b7a457396c7674b04db10e6"
-    },
-    "52bda6cbab224899845e66e0474cdefc": {
-      "label": false,
-      "text": "sees the formula graph, the chip calculates the formula, able to \"survive\" thanks to its connection to Edit, develops a parallel personality and affords her abilities greater than she ever imagined...",
-      "id": "52bda6cbab224899845e66e0474cdefc"
-    },
-    "435aabe68c294963a05e090d479582bc": {
-      "label": false,
-      "text": "Aanandam is a 2016 Indian Malayalam campus musical film written and directed by Ganesh Raj in his directorial debut. Vineeth Sreenivasan produces the film under the banner of Habit Of Life with Vinod Shornur under Cast N Crew.",
-      "id": "435aabe68c294963a05e090d479582bc"
-    },
-    "f96313d0087e4941a359783634ef9e86": {
-      "label": false,
-      "text": "The remarkable story of The Weather Underground, radical activists of the 1970s, and of radical politics at its best and most disastrous.",
-      "id": "f96313d0087e4941a359783634ef9e86"
-    },
-    "f63e4502791a409fa2d750687d3841eb": {
-      "label": false,
-      "text": "A young widow on a trip to the backwoods stumbles upon the operation of a gang of drug smugglers. They attempt to kill her in order to keep their operation a secret, but she turns out to be more resourceful than they thought, and starts to turn the tables on them.",
-      "id": "f63e4502791a409fa2d750687d3841eb"
-    },
-    "108ac02949324b02bdcbe4c7a77bacdc": {
-      "label": false,
-      "text": "The story of a young Marine, fresh from Camp Pendleton, who is forced to confront the complexities of adulthood and a volatile home life during a four-day Thanksgiving leave.",
-      "id": "108ac02949324b02bdcbe4c7a77bacdc"
-    },
-    "44fc412246964b2393fa0035ff093a00": {
-      "label": false,
-      "text": "Exploring the rough and tumble world of hockey, Academy Award winner Alex Gibney (\"Taxi to the Dark Side\") looks at the world of the NHL enforcers and specifically the career of Chris \"Knuckles\" Nilan who helped the Montreal Canadiens win the Stanley Cup.",
-      "id": "44fc412246964b2393fa0035ff093a00"
-    },
-    "409350c111af4ba3a94c842b797ddb95": {
-      "label": false,
-      "text": "Two fishing fanatics get in trouble when their fishing boat gets stolen while on a trip.",
-      "id": "409350c111af4ba3a94c842b797ddb95"
-    },
-    "d48d8f3b5a524ecea69bae718d1f1513": {
-      "label": false,
-      "text": "A willful young boy follows his just as obstinate grandmother in a journey across Iraq, determined to discover the fate of her missing son, Ahmed's father, who never returned from war.",
-      "id": "d48d8f3b5a524ecea69bae718d1f1513"
-    },
-    "283e96de5b474240a044c50dbc2551fb": {
-      "label": false,
-      "text": "A group of people are sitting in a theatre watching a movie when one realises that the woman on the screen is her. (IMDb)",
-      "id": "283e96de5b474240a044c50dbc2551fb"
-    },
-    "516d0f2f3a854a97a87c64db19a89fac": {
-      "label": false,
-      "text": "of the fake prediction. Fantastic swashbuckling adventures in a 18th century setting, with a light criticism of the war and the mighty.",
-      "id": "516d0f2f3a854a97a87c64db19a89fac"
-    },
-    "c2f55710669b40aa937625fe0ab04065": {
-      "label": false,
-      "text": "famous for his reputation as a Don Juan, to seduce C\u00e9cile and emotionally destroy her. While on his mission, Valmont gets sidetracked when he goes to visit his aunt and falls for Madame Tourvel, a virtuous, married woman who knows of his womanizing ways, but that only makes the challenge more exciting to Valmont. Together, Madame de Merteuil and Valmont make a dangerous team and they will stop at nothing when it comes to matters of the heart.",
-      "id": "c2f55710669b40aa937625fe0ab04065"
-    },
-    "ba0261b2ee3244d29bb3a8c6d77195a6": {
-      "label": false,
-      "text": "sees the formula graph, the chip calculates the formula, able to \"survive\" thanks to its connection to Edit, develops a parallel personality and affords her abilities greater than she ever imagined...",
-      "id": "ba0261b2ee3244d29bb3a8c6d77195a6"
-    },
-    "5e724fbde8ee44d9a8fc87a6e6667f01": {
-      "label": false,
-      "text": "telling the story about people who despite all obstacles strive for their goal.",
-      "id": "5e724fbde8ee44d9a8fc87a6e6667f01"
-    },
-    "557eba5ebfc9467a9d88688afed41354": {
-      "label": false,
-      "text": "A young playboy who learns he has one month until he becomes infertile sets out to procreate as much as possible.",
-      "id": "557eba5ebfc9467a9d88688afed41354"
-    },
-    "aa20e22fbe96487d8ee1223a6ef4da0b": {
-      "label": false,
-      "text": "Set in modern times, Alex finds King Arthur's sword Excalibur and must prove himself worthy of it.",
-      "id": "aa20e22fbe96487d8ee1223a6ef4da0b"
-    },
-    "bea56d34f6df408c9ec9653b17a90a93": {
-      "label": false,
-      "text": "Kostis is a 40-year-old doctor that finds himself in the small island of Antiparos, in order to take over the local clinic. His whole life and routine will turn upside down when he meets an international group of young and beautiful tourists and he falls in love with Anna, a 19-year-old goddess.",
-      "id": "bea56d34f6df408c9ec9653b17a90a93"
-    },
-    "e61a3251720d425c9f4770cb4b11d2d9": {
-      "label": false,
-      "text": "Friends on a weekend excursion take a path into a forest that leads to death and destruction.",
-      "id": "e61a3251720d425c9f4770cb4b11d2d9"
-    },
-    "5471008376cf44518f2ff1f67f057c08": {
-      "label": false,
-      "text": "Mr Bournelis suggested all 30 lineal metres of blockwork should be removed and replaced, which would require removing and reinstalling the fence. The total cost of his suggested method of rectification was said to be $14,650 for each unit, giving a total cost of rectification of $29,300.",
-      "id": "5471008376cf44518f2ff1f67f057c08"
-    }
-  },
-  "version": 27,
-  "description": "Negative sentiment"
-}

data/concept/lilac/positive-sentiment/concept.json

@@ -1,564 +0,0 @@
-{
-  "namespace": "lilac",
-  "concept_name": "positive-sentiment",
-  "type": "text",
-  "data": {
-    "0": {
-      "label": false,
-      "text": "Starting To Be Annoyed By Becky...: I'm not sure why I keep reading these books, but I guess it's because I've read the first two so I'll keep reading the rest of the books. In the first book, I really found it amusing. I was a little annoyed by the fact that Becky couldn't stop spending, but then again that's why she is called a Shopaholic. In the second book, I felt more of the same it was just magniifed more. Now in the third book, I'm just down right annoyed by Becky Bloomwood. In this book, she wasn't going on crazy shopping sprees, just planning two different weddings because she was afraid to tell each person and because I feel she's really selfish. Still, I read the book because I wanted to see how she could get herself out of another situation. I will say that I love her friends Suze and Danny, her client Laurel and her husband Luke. Maybe that's why I keep reading. I will read the next book, but I'm sure I'll be just as annoyed when I'm done.",
-      "id": "0"
-    },
-    "1": {
-      "label": false,
-      "text": "the cover is fine - the pool is horrible: The entire pool was horrible. The cover was fine once we got it on, but we finally got rid of the pool after 2 weeks because it was so hard to set up and keep clean.",
-      "id": "1"
-    },
-    "2": {
-      "label": true,
-      "text": "Good album, not their best.: This album is probably the most marketable and radio friendly of all of dashboard's albums. For the peripheral listener it may be the right one to get to introduce you to this band. But as a Dashboard fan of 5 or so years I truly hope they return to their original sound for their next work. Not for the listen-ability but for the show. To this day the fans react best to the songs from \"Places\" or \"A Mark, A Mission.\" I recommend this album to everyone but I also recommend any and all of their other work.",
-      "id": "2"
-    },
-    "3": {
-      "label": false,
-      "text": "This is a horror novel, right?: Never one to pass up any vampire novel, I purchased Sips because the description seemed interesting. Vampires, Marquis de Sade, fetishism, yada yada yada. If this is a comedy, I give it 4 stars; however, I'll give it 1 star as a horror novel. Sade was rather boring; I would think a character as intense and multi-faceted as the Marquis de Sade would make for a more interesting vampire. The writing style isn't too bad, but overall I found the characters to be mildly amusing at best. The plot was thin, the end was anti-climactic, and the vampires were not very frightening. The book had little suspense, and it leaves a mile-wide opening for a sequel at the conclusion. I would, however, like to see something more of the vampire mutants lurking in the graveyard. They were the most riveting of any of the characters.",
-      "id": "3"
-    },
-    "4": {
-      "label": true,
-      "text": "Superb mix of global non secular musical denominations: I first heard Ms. Pook's music on the \"Eyes Wide Shut\" soundtrack (the masquerade ball scene) and was blown away; if ever there was a necessity for music to permeate a scene in a film this was it. She incorporates a blend of the traditional songs from faiths across continents and mixes them, for lack of a better comparison than similar to your quintessential raver d.j. (though these are better and definitively more original :) \"Oppenheimer\" is my favorite, and if you let the last track run for a few minutes a portion of the song will play once more. I can't wait to hear more of her stuff - these hymns are awesome.",
-      "id": "4"
-    },
-    "5": {
-      "label": true,
-      "text": "A moving and suspenseful story!: For anyone familiar with the occult, this book is likely to raise hairs on the back of your neck as you read. Even if you're not, the storyline is suspenseful and fascinating, and the characters evoke great sympathy and admiration. An excellent read.",
-      "id": "5"
-    },
-    "6": {
-      "label": true,
-      "text": "Simple & Easy to Use - A Practical approach to eating out: This guide is extremely to use. It provides sample menus that you'd see at Chinese, Indian and Thai restaurants. Then you are provided with descriptions of each dish and how it is prepared and the ingredients used. From there you are provided with specific considerations as to how the preparation or ingredient list may affect you if you have Gluten or Allergen issues.This book is the size of a passport and very organized and well written. The Chinese, Indian and Thai Cuisine Passport is perfect for making choices while traveling, or while dining at your favorite local restaurant.",
-      "id": "6"
-    },
-    "7": {
-      "label": false,
-      "text": "Being Fair....I am a fan of hers: and I really enjoyed her previous works, more than I could have ever imagined, but this record is horrible. The songs are trite, the lyrics are incredibly boring, indulgent and immature. The music is pop staple, with forgetable melodies and repetative chorus lines, I feel as if the studio wrote the entire album for her while she was sleeping, this just doesn't speak to any of her previous works at all. This album fits on the same shelf with a Nickelodeon-themed CD. Instead of heading in the direction of an artist like Alanis Morrisette, she is going backward and joining the ranks of Hannah Montana and the Naked Brothers Band. She is a great artist and her first two records are amazing. She is better than this CD and I am looking forward to her next effort.",
-      "id": "7"
-    },
-    "8": {
-      "label": false,
-      "text": "Sucked: I thought the DVD sucked tremendously. It was very boring and if I could, I would return it for a refund. There was only one \"small\" clip of Dylan himself. I'm very disappointed.",
-      "id": "8"
-    },
-    "9": {
-      "label": true,
-      "text": "Excellent product: Easy to install. If you have a newer furnace you probably do not need the swail switch as the HE220A comes with a Humistat which can be connected to the furnace. They recommend the Honeywell 32005847-001 Installation Kit, Bypass which is a little pricey and you can probably buy the pieces of this kit cheaper individually from Home Depot or Lowes or ACO as well as the filters.",
-      "id": "9"
-    },
-    "10": {
-      "label": true,
-      "text": "Very happy.: I am very happy with this trashcan. I was unable to find one in the stores to fit the space in my cabinet, but this one does the job. It is very sturdy and looks like it will put up with years of use.",
-      "id": "10"
-    },
-    "11": {
-      "label": false,
-      "text": "These aren't Throughbreds!: This makes me so mad. All these new authors are coming and changing the series. Nothings the same anymore and the plots are repeditive. Don't even bother reading these books until #32 these are like a different series. I don't know excactly what's happing but these new authors suck!",
-      "id": "11"
-    },
-    "12": {
-      "label": false,
-      "text": "Large and slow are a bad combination.: I bought this TV and returned it a week later, because it blurred so badly with motion that sports were unwatchable. I ended up buying a smaller Sony XBR4, and I have none of the issues (plus the picture is far, far better).This has nothing to do with 60 vs 120Hz. That is more important for DVDs and Blu-Ray signals that are 24fps (which doesn't divide evenly into 60 but does for 120). The LT52133 has an 8ms response time, which is extremely slow. A decent LCD should be 5 or lower.If you want an LCD, choose speed and quality over size. If you want size and quality but want to spend less, buy a plasma. Don't buy a big, cheap, slow LCD!I gave it 2 stars because I like the interface and remote.",
-      "id": "12"
-    },
-    "13": {
-      "label": false,
-      "text": "Skip it: This movie is very poorly written and the result is not distressing, just lame. The actors do their best but from very early on it is obvious that the material gives them nothing to work with. Fans of Colin Firth will experience a certain dim level of enjoyment. Minnie Driver is a treat but her character is no better written than the others. Vermont locations are worth something. With one or two moments of exception it's neither comedic nor romantic.",
-      "id": "13"
-    },
-    "14": {
-      "label": true,
-      "text": "Belive it i went to the concert?: hi everyone let me tell you i went to the concert i was amazed with what i saw cher was awsome i tell you buy the dvd. as i sat in front of the stage cher was doing a great job to us the she is living proof . So i urge you to buy it?",
-      "id": "14"
-    },
-    "15": {
-      "label": true,
-      "text": "Vale la pena.: En este libro se narra de una forma muy interesante la vida de una familia en particular. Lo que mas me gusto de este libro fue la manera en que la autora describe a lo largo del libro las personalidades de los sujetos envueltos en la novela; que vienen a ser muy distintos y extremos, lo cual, intensifica el drama... Definitivamente es un buen libro y lo recomiendo a todos.",
-      "id": "15"
-    },
-    "16": {
-      "label": true,
-      "text": "Nummie Children's story: I ordered this book for our grandchildren. Two boys 5 & 3 and a 4 month old girl. All love the story. The mouse is determined.",
-      "id": "16"
-    },
-    "17": {
-      "label": false,
-      "text": "Seem to be alone on this one: Looking at the other reviews, I seem to be the only one that was disappointed with this book. The content is too babyish in most of it for older tweens and the more \"grown up\" content would be over a younger tween's head. I had a quick read through and with every paged turned, I thought duh. I'll be looking around for another book shortly.",
-      "id": "17"
-    },
-    "18": {
-      "label": true,
-      "text": "Best yet: by far the best EA game yet. I especially like the easy controls and kick - a graphics. the playbook is extremely accurate and detailed. Also the fight songs and cheerleaders were a nice touch. this is an excellent game and worth checking out.",
-      "id": "18"
-    },
-    "19": {
-      "label": false,
-      "text": "washed out: A bit like Simply Reds version of the Valentine bros hit \"Moneys too tight to mention\" - this cover version has nothing of the driving energy that characterised the original recording.",
-      "id": "19"
-    },
-    "20": {
-      "label": true,
-      "text": "great water bottle: I love this bottle it is great. I like knowing it is non toxic and it just works very well. You can have it full and lay it down and it doesn't leak at all.",
-      "id": "20"
-    },
-    "21": {
-      "label": true,
-      "text": "Nice goggles: I am pretty happy with these goggles. They work well during swim workouts in the pool. I do notice a little bit of fogging from time to time. I had hoped to wear them during an upcoming triathlon, but based on a few instances where they slipped a little in the pool I am concerned that they won't be secure enough. I will keep using them in the pool, but will likely get different ones for open water races.",
-      "id": "21"
-    },
-    "22": {
-      "label": false,
-      "text": "aaahhh nnnoooooo!: Hopefully the last film in one of the worst horror trilogys ever made. This series pretty much ruined the horror film for years to come, for one its too self aware, thats incredibley annoying, second everyone acts like they are on Friends or some sitcom. The acting is just plain bad and unconvincing. Now the gore, if you're going with material this weak you should load it up with disgusting violence, is there any in the Scream series? No.Everyone went to see this movie just to see who THE KILLER is. This movie sets low standards to be met, you expect alot of people to die, one shock, then we find out who the killer is, then you go home. Every horror film being made today is like that, there's nothing new or exciting or risk taking, its the same stuff over and over and people are laping it up like dog food.This film is what you've come to expect, bad acting, some people die and we eventually find out who the killer is and all is merry and well. Pathetic.",
-      "id": "22"
-    },
-    "23": {
-      "label": true,
-      "text": "A classic of its kind: This movie is a classic of its kind and much better that a lot of movies, that followed. It is not one of the best, but it still deserves five stars...",
-      "id": "23"
-    },
-    "24": {
-      "label": false,
-      "text": "Nice suite, but Virtual PC 7 disappoints on my G5: I purchased the upgrade since I'd already bought both Office v.X and Virtual PC 6.1 last year.The biggest letdown is that Microsoft's promised support for the G5 is nearly non-existent. I have a dual processor G5 with an ATI Radeon 9800 card (Apple), and after trying to install Virtual PC 7 three times, I cannot get a VM to work. It did install (and work) flawlessly on my G4 Powerbook. Googling for reviews finds it's very hit or miss, but if (when) it misses, you'll regret investing the extra $$$ in an immature product.",
-      "id": "24"
-    },
-    "25": {
-      "label": false,
-      "text": "Okay player, don't expect a miracle: I bought this DVD player at Circuit City earlier this yr for about a $100. I hooked it up to a 47\" Vizio LCD (which by the way has an awesome picture) using a HDMI cable. After fine tuning this product, I was very, very, very diasppointed. The picture was very \"grainy\" (lots of pixels). I have a $35 DVD player that only utilizes an s-video cable that produces a much more crisp picture. Be warned, the picture stinks.",
-      "id": "25"
-    },
-    "26": {
-      "label": true,
-      "text": "A revelation of the science of consciousness evolution and all natural growth: Here is a readable and fascinating account of the development of the new science of chaos theory, the only body of ideas that describes how the natural world as experienced by human beings emerges out of basic quantum processes. The different explorers and innovators of the new science are introduced in a personable way that will enchant the interested reader.",
-      "id": "26"
-    },
-    "27": {
-      "label": false,
-      "text": "Don't say that I didn't warn ya' !: I'm absolutely convinced that Delbert McClinton had no controlover the release of this CD. I rated it 1 star simplybecause there is no 0 star rating ! In actuality , I am not certain that the vocalist on this recording IS Delbert McClinton. Only on the Mr. Pitiful track is there any similarity at all to Delbert's voice. This is the perfect CD for someone with money to burn who would like to have a recording of a 1960's garage band recorded in a garage and who should be working in a garage ! Delbert fans...run fast and run far away from this ! END",
-      "id": "27"
-    },
-    "28": {
-      "label": false,
-      "text": "This item is not available: I ordered this unit on February 7th. Every time I checked back on the status of the order, it read \"not shipped\" and the estimated shipping date got moved out. I really don't think this unit is avaialble from the company anytime soon. I cancelled the order.",
-      "id": "28"
-    },
-    "29": {
-      "label": false,
-      "text": "I used to like ABBA...: I used to like ABBA, until I saw Mama Mia! A horribly disjointed musical, where songs feel contrived to fit into the story; a story that doesn't seem to come together. Individual songs are usually done alright, but don't segue from one to another very well.The cast butchered several of the songs, but especially S.O.S, Take A Chance On Me, and anything where Pierce Brosnan sang. On a side note, I also counted at least two violations of Chekov's Gun. And finally, I think it has a bad moral message. Which you only recognize if you manage to sit through the whole thing.If there is justice in the world, cast members without established careers won't get to have them as punishment for the worst movies I've seen since The Talented Mr. Ripley.",
-      "id": "29"
-    },
-    "30": {
-      "label": false,
-      "text": "A complete disaster!: If you're like me, you probably wanted to check out this movie because it sounded like it really could be an excellent supernatural Gothic horror tale full of goblins and wicked things alike. Well, don't make the same mistake I did and actually watch it. It's horrible. Terrible. An honest to goodness waste of film. The acting is wretched, the film quality is rotten (it actually looks twenty years older than it is), and the plot is thin, weak, and does not give you what it's supposed to. The only reason I bothered to give this film 1 star is because of Alexis Arquette -- he's great looking, but should have left this film out of his career.",
-      "id": "30"
-    },
-    "31": {
-      "label": true,
-      "text": "beautiful detail: I just purchased these Dover COloring Books for my mother and she loves them. The detail is out of this world and the variety of colors you can use are only limited by your inagination. HIGHLY RECOMMENDED!",
-      "id": "31"
-    },
-    "32": {
-      "label": false,
-      "text": "Very disappointed: I looked forward to getting this movie as I had heard many good things about it but it was nothing like I had imagined or been led to believe. There is very little actual history in it or real Christian experience except for the background because the main focus is a soap opera style romance and caricature figures. I agree with the reviewer who described it as a mixture of \"tawdry Hollywood sex\" somehow interspersed with a vague nod to Christianity. The only decent scene was the arena scene where the Christians are going to their deaths singing hymns - but that's not enough to make it a great or even a good movie. Not personally to my taste anyway.",
-      "id": "32"
-    },
-    "33": {
-      "label": false,
-      "text": "Unreliable minikit: I bought this minikit because it got good reviews and it would be perfect for my purposes. However it switches on and off whenever it wants, it looses contact with the phone. Very often the on/off button works only in a horizontal position (?) I use a Treo 650, which is on the compatible phone list. When I contacted Parrot, they said it wasn't (?) At last I opened the unit, but there are no moving parts inside except the micro switches. It is giving me a headache, so I will go searching for an alternative.",
-      "id": "33"
-    },
-    "34": {
-      "label": true,
-      "text": "A Christmas Classic!: This is surely one of the best classical Christmas recordings available. Don't buy the older version, as the quality of this recording is excellent. This is one of those \"Every Christmas - Can't have Christmas without\" recordings.",
-      "id": "34"
-    },
-    "35": {
-      "label": false,
-      "text": "too narrow: These were the narrowest pair of D size shoes I have ever tried on. I don't care how nice a shoe looks. If it don't fit it just don't fit.",
-      "id": "35"
-    },
-    "36": {
-      "label": false,
-      "text": "Lack of extension: This earphones lack a descent extension cord. ITs very small cable, but its of good quality. Sadly, cord its too short, and the extension is useless.",
-      "id": "36"
-    },
-    "37": {
-      "label": true,
-      "text": "Easy-Reading: This is the 3rd Southern Sisters Mystery I've read. They're easy, fast and funny murder mysteries, with lots of cute family stories intertwined in the intrigue.",
-      "id": "37"
-    },
-    "38": {
-      "label": false,
-      "text": "it'd be great if it worked like it was supposed to: for the first 30 seconds it was lovely, but i believe that either the motor isn't powerful enough to keep the shaft rotating smoothly or 3 AA batteries just don't provide enough juice for the motor to work more than 30 seconds. it was a nice idea, but i'm rather dissapointed. the jelly material is somewhat difficult to maintain also. i think if it were hooked up to a larger battery pack it'd be WONDERFUL... which i think i may have a macgyver friend with a knack for electronics attempt to do for me.",
-      "id": "38"
-    },
-    "39": {
-      "label": true,
-      "text": "Not Hornby's best but still good: I loved About a Boy and really, really loved the sardonic wit of High Fidelity. About a Boy is much deeper but just as cynical. Maybe even more so. The characters are richly drawn and just complex enough to keep the reader wanting more. Good read, but best to take some time with this one. Not recommended for a summer beach read.",
-      "id": "39"
-    },
-    "40": {
-      "label": false,
-      "text": "A Disappointment: As with most Taunton Press publications, the illustrations and photographs in this book are spectacular and the organization and layout is superb. Nonetheless, I found this book disappointing. It lacks both depth and breadth. I had hoped for a detailed review of wood joinery including some of the more unusual joinery found in Japanese woodworking. This book, however, is targeted more toward the beginner. Even so, it does not cover the details and \"tricks\" of even the most basic techniques in sufficient detail to allow beginners to easily reproduce them. Consequently, it is unclear who this book was written for - not the beginner as it lacks depth, and not the advanced woodworker as it lacks breadth. Far more effort appears to have been put into appearance and organization than in content.",
-      "id": "40"
-    },
-    "41": {
-      "label": false,
-      "text": "Horrible. Don't do it!: Great price for the item when a 6' one of these at Best Buy is $20. Thing is, the one from Best Buy fits in the outlet and stays there. This cord fits very loose and does not connect. I bought 2 of them, neither did what they were suppose to.As much as I hate to say it, but, buy the more expensive one. At least it works.",
-      "id": "41"
-    },
-    "42": {
-      "label": true,
-      "text": "Given as a gift...: Given to my best friend as a gift. She loves it. Her fiance enjoys making coffee for her in the mornings. :)",
-      "id": "42"
-    },
-    "43": {
-      "label": true,
-      "text": "Love the ring.: This is a nice ring. I was worried it out be thin and cheap looking, but it's not. It's a very pretty stylish ring. Go for it.",
-      "id": "43"
-    },
-    "44": {
-      "label": false,
-      "text": "Beautiful writing Marred by One-Note Characterizations: How could Kingsolver have ruined her book with such an obvious error? Nathan is a strident paper doll that flattens the whole story. Just as bad, the author has all the narrators using the same ironic tone to decribe him, deadening their voices as well. At the same time, Kingsolver doesn't have the guts to show him doing something trully terrible. I don't trust an author who can't let the reader make up his own mind, and as a consequence I couldn't trust her views about ANYTHING in the story. I'm giving this two stars for her descriptions of the African landscape, and that is all.",
-      "id": "44"
-    },
-    "45": {
-      "label": false,
-      "text": "Much worse than any cordless phone I've ever had: This phone cuts out only 2 rooms away from the base station. There is static noise, and callers on the other end complain about sound quality. I can't go into the garden, which used to be no problem with my old 900 MHz phone.",
-      "id": "45"
-    },
-    "46": {
-      "label": false,
-      "text": "Waste of time & money: The first Hangover was not too bad, this one was just terrible. The acting is bad, the script is bad, everything about this movie was just bad. Do yourself a favor, don't buy this movie as it is a total waste of time and money.",
-      "id": "46"
-    },
-    "47": {
-      "label": false,
-      "text": "Did Not Work For Me!: Impressive You Tube Video (Like a Sci-Fi Fantasy). In reality it's a high speed Easy Out so unsurprisingly it broke faster than an Easy out. This product did not work for me. The drill part did not drlil, the puller part did not pull. It was a total zero.",
-      "id": "47"
-    },
-    "48": {
-      "label": true,
-      "text": "Excellent book, long overdue.: From a very long time women were told that looking good was of utmost importance. This was without regard to health or fitness and how age affected these parameters. Witness the whalebone and other types of corsets, the spike heeled shoes and the numerous weight loss programmes on the market (some of which are downright dangerous). Now there is a book, backed by solid research, that allows women of all ages to remain fit and healthy for a lifetime. I am certainly going to recommend this book to all the women I know.Bentley Norville",
-      "id": "48"
-    },
-    "49": {
-      "label": false,
-      "text": "not an all star: Not a practical guide in this collecting age. Does NOT have a comprehensive list; meaning it does NOT cover all manufacturers and, more importantly, for the ones it does, only provides listings of the base set. That means no insert or variation pricing whatsoever. Also, no oddball or minor league issues are listed. Generally speaking, unless you are collecting base sets prior to the advent of inserts and alternate versions of the base set, this guide is fairly useless.",
-      "id": "49"
-    },
-    "50": {
-      "label": false,
-      "text": "Again, second rate city, third rate writer: Just another example of Mr. Lindberg's pitiful attempt at exhibiting a strong expertise on a subject with which he is clearly obsessed. Don't waste your time with this book, either. It is poorly written and fails to engage the reader. You might consider using this book and the first book he wrote on the same subject, as a pair of bookends. That is about all they are worth.",
-      "id": "50"
-    },
-    "51": {
-      "label": true,
-      "text": "Reality: No one should need to convince you to buy this book, you should just do it! It's so well written and worded and brings you right to the heart of a sexual reality that most people like to pretend doesn't really live and breath in their fair cities. I never again want to hear someone bad mouth a working girl for what she does. I will and do now however look at men with a curious eye wondering if they are depraved peep show window lickers :)",
-      "id": "51"
-    },
-    "52": {
-      "label": false,
-      "text": "Bummer: Visual effects and Battle footage were great...the other 85% of the movie was just lousy fluff...",
-      "id": "52"
-    },
-    "53": {
-      "label": true,
-      "text": "The spark of idependence: Filled with the independent spark that made us all love life at one point or another. A fun, introspective and nonsensical movie that sticks with you.",
-      "id": "53"
-    },
-    "54": {
-      "label": true,
-      "text": "What I expected from Mirman's website. Funny. Funny. Russian.: lol, gotta love Eugene. Even when his audience doesn't initially laugh, he gets in a good zinger at himself and they laugh at that. He's witty without being condescending, and uncomplicated without seeing contrived. However, if you're not a fan of irreverant humor, this may not be for you.",
-      "id": "54"
-    },
-    "55": {
-      "label": false,
-      "text": "Do not...repeat...do not bother!: It is not often that I offer a negative review but this compilation while attractive does not deliver at all.The foot massage gizmo is awkward and uncomfortable.The pumice stone leaves rough splinter like skin.The foot scrub doesn't reall scrub.The rotary action tool has five heads, none of which work well and you must hold the switch in place or it turns off. It is cumbersome and ineffective.The one star was initially given for a foot brush (which later lost its bristles very easily as I update the review) and a sweet smelling foot repair balm.Don't waist your money. Soak your feet and invest in an inexpensive German Titania file, smooth and coarser side, or a like product. It will last for years.",
-      "id": "55"
-    },
-    "56": {
-      "label": false,
-      "text": "Not Sandra's Best: Ms. Brown has written better romance novels. Don't give up on her if this was your first Sandra book.The feeble female lead struggles with a 15-year crush that walks back into her life. The smug male lead acts like a jerk through most of the novel. The romance scenes grapple to muster up passion but fall short. Both of the main characters bothered me; my favorite character was the 17-year old.A quick read...about 4 hours (with interruptions) for me...but probably not worth it.",
-      "id": "56"
-    },
-    "57": {
-      "label": true,
-      "text": "Impressed: Lots-O-Fun. Wood and glass toys are high quality and are a good fall back for the kids to play with they are \"bored\". Would buy again.",
-      "id": "57"
-    },
-    "58": {
-      "label": false,
-      "text": "Light turned on by itself 3 times: The installation was easy. I used it for a week, everything worked fine, EXCEPT the light it connected to turned on by itself 3 times so far, with no one near to either one of the switch. Not sure whether it is a defective unit, or this product is too sensitive to noise. I'm returning this product and will just install a regular switch instead.",
-      "id": "58"
-    },
-    "59": {
-      "label": true,
-      "text": "good battery: I feel kind of silly writing a review for a battery, but have to say that these last a LONG time. Work very well.",
-      "id": "59"
-    },
-    "60": {
-      "label": true,
-      "text": "Even a Woman finds it funny: Yes, even a woman finds \"Married to Mommy\" funny. The book gets you laughing aloud when it is trying to make fun of \"Mommies\". The truth is that it really is making fun of the stupidity of men and their simple basic needs of sex, getting out of work, and beer. Of course, the truth is always funny.A definite MUST for any woman, married or not. We will now know all the secret tricks the men try to use on us.By the way, I am NOT a MOMMY!",
-      "id": "60"
-    },
-    "61": {
-      "label": true,
-      "text": "Gungrave...not quite what you might expect: Those thinking this is another version of Trigun will be disappointed. Gungrave is actually a lot deeper and more complex. The lead is short on dialouge, but the story has more depth and character development than most anime. The first DVD is more about the main character's past than about the reanimated killing machine he's become, but it definitely leaves you wanting more.",
-      "id": "61"
-    },
-    "62": {
-      "label": true,
-      "text": "Error in product description: It's great in every way. However, if you'd prefer a digital tuner (as I do), then you might need to look further. The product description boasts a digital AM/FM tuner, but it's disappointingly an analog AM/FM tuner.Overall - especially for the price - I think it's pretty good.",
-      "id": "62"
-    },
-    "63": {
-      "label": true,
-      "text": "good phone but not as user friendly as it could be: Battery life is very good. Phone has good range. My only complaint is it's to involved to get your message from the handset.",
-      "id": "63"
-    },
-    "64": {
-      "label": false,
-      "text": "Big waste of money (and space in my house!): My 5 year old son wanted this so bad, but when we got it for him, there were so many pieces to put together that didn't fit together well, he never played with it. It just sits on our floor in many pieces taking up toy space! What a waste!",
-      "id": "64"
-    },
-    "65": {
-      "label": true,
-      "text": "Don't want to take it off: Very satisfied with an earlier purchase of this Bali bra model, I was just as pleased with the new one. Very comfortable, well made and a good neutral color. It will be my next choice, too.",
-      "id": "65"
-    },
-    "66": {
-      "label": true,
-      "text": "Fantastico: If anybody who's into rock music is ever looking for a band to keep you on your toes, this is the band. I've been a fan for 10 years now, and no album has ever sounded like any of their previous albums. This disc is fantastic with such a variety of styles, as are the previous releases, even back to the Rainbow Butt Monkey days.",
-      "id": "66"
-    },
-    "67": {
-      "label": false,
-      "text": "too much visual: There are far too much designs, visuals, colors, etc in the book - this is highly distracting, as TV screen can be...By way of example (among so many...), what is the use of colors with the three squares of the Pyth. theorem???? this is as useless as writting 2+3=5 with 2 in blue, 3 in red and 5 in yellow...I wish I had purchased the 2nd edition, which according to reviews was closer to what I was looking for.",
-      "id": "67"
-    },
-    "68": {
-      "label": true,
-      "text": "Aretha's First Arista Release Showed Pleasures to Come: After a long and musically satisfying career with Atlantic, Aretha severed her ties with that company and moved under the wing of Arista's Clive Davis. With the start of the 1980's, Aretha was looking for new territory to conquer and almost succeeded with this mixed bag.\"United Together\" is a fine tune that benefits from beautiful orchestral arrangement that is matched by Aretha's superb vocal instrument. The remake of \"Can't Turn You Loose\" allows Aretha to show why she is the Queen of Soul\" for she really belts this one out. Another cover, that of the Doobies' \"What a Fool Believes,\" is an interesting interpretation. The final cut \"School Days\" appears to be \"autobiographical\" for every girl growing up in the fifties.Although not as strong as her Atlantic work, \"Aretha\" is still a suitable addition to the artist's discography.",
-      "id": "68"
-    },
-    "69": {
-      "label": false,
-      "text": "Misguided Purchase: The photo and description do not reflect the product. The screen panel kit I received was white. What a huge inconvenience during a time-crunch.",
-      "id": "69"
-    },
-    "70": {
-      "label": false,
-      "text": "Banacek: My husband and were looking forward to seeing this series.The first show was SO boring, we finally just quit watching it.Actually, we haven't gotten around to watching anymore. I guess we were afraid of a repeat.Maybe that was just once, I hope!",
-      "id": "70"
-    },
-    "71": {
-      "label": true,
-      "text": "JDT: Uncle Tupelo is without doubt one of the most under appreciated groups of the 90's. Anodyne, like each of the three albums that came before it, has everything that a remarkable recording requires: great songs, honest lyrics, and artists who really care about the music they are making. Like the best of Dylan and Springsteen, the songs are about real people with real troubles and joys. When you hear them you know they are coming from the heart. The songs contributed by Jay Farrar and Jeff Tweedy are easily differentiated by the voacls, music, and lyrics. What makes this record interesting is how well these unique sounds compliment each other. The union is seamless.",
-      "id": "71"
-    },
-    "72": {
-      "label": true,
-      "text": "Well Worth Reading: First a confession: Miriam Wasserman was my mother. However, she published several books, but this is the only one I really found useful. She walks the reader through the New York City school system and the attitudes of different groups involved in the system back in the 1960s. This includes parents, teachers and administrators. Her view is that the further away one got from parents and students, the more prestige one had. She meticulously describes the teachers' strike of 1968 against \"community control of schools\", a strike of which she is extremely critical. She explores the racism that was involved in this strike, including using quotes from striking teachers, etc. It should be emphasized that the author was pro-union all her life, so her views don't stem from an anti-union bias. The book also covers the high school student rebellion which coincided with and followed the strike.",
-      "id": "72"
-    },
-    "73": {
-      "label": true,
-      "text": "compact and loaded: I bought this phone after reading the cnet reviews and really liked it. It looks small and really compact. I like the camera pics at 2 mega pixel and bright flash. The mp3 player is crisp. The headset that comes along delvers amazing fM radio. I think my phone is not very loud and you have a problem when you are around a noisy crowd. I just bought this phone again for my cousin. He likes it too. Almost forgot the display is very good.",
-      "id": "73"
-    },
-    "74": {
-      "label": true,
-      "text": "Outstanding text!: Brooks/Cole should keep this text in their catalog for ages! It is well-written, examples are generally quite clear, vocabulary is introduced well, and the exercises develop real skills, rather than simply be busy-work. One of the best calculus books ever!",
-      "id": "74"
-    },
-    "75": {
-      "label": true,
-      "text": "Excel 2003 Bible: Very good source of information. I will most likely buy other books in this series.",
-      "id": "75"
-    },
-    "76": {
-      "label": true,
-      "text": "Tasting is Believing: Gluten-free breads used to have a gritty texture from the rice flour, and were too soft for sandwiches. Bette Hagman uses garbanzo/fava bean flour, sorghum flour, tapioca flour, and corn starch to create breads which have a similar texture to wheat flour breads, and the flavors of her breads are fabulous.My BF bought me this book and a great tasting beverage to drink it with. Since he knows I quit coffee recently, he's been really wonderful helping me in cope with my mood swings. S o y f e e is made from soy beans that is roasted just like coffee. I enjoy the taste and don't miss coffee one bit. Buy it online at www.s o y c o f fee.com.This is a 'must have' for anyone baking gluten-free. I think all of Bette Hagman's books are wonderful and a must for those with gluten intolerance.",
-      "id": "76"
-    },
-    "77": {
-      "label": true,
-      "text": "5 stars for the show, no stars for the \"Collector's Edition\": I was really looking forward to getting this Collector's Edition and see what extras were added. I knew it wasn't a lot - just a mini-book and a documentary - but I figured it would be packaged in a cool way.Wrong.As others have already mentioned, the Collector's Edition is *literally* theAvatar: The Last Airbender - The Complete Book 1 Collectionslipped into another cardboard box, with a little booklet and DVD in an envelope (not even a case!) wedged in. It's really disappointing; it would have been so easy to create a quality Collector's Edition but the studio couldn't be bothered, I guess.",
-      "id": "77"
-    },
-    "78": {
-      "label": true,
-      "text": "sula scottcampos: Sula, a book that talks about the issues of being a black women is a really good novel to read.One of the reasons I recommend it is because of its realism and its themes - death, sex, friendship and poverty.I also think that its characters are very good, its easy to identify with one or both of them. I really recommend this book to anyone who enjoys good literature.",
-      "id": "78"
-    },
-    "79": {
-      "label": true,
-      "text": "Fantastic! It's a must-have for girls!: I hated razor, tried shaving but it did not work for me. Shaving made the hair grows thicker and faster afterwards, plus the roots are impossible to be getting rid of. After reading the reviews, I ordered it to try, I used it for once and already fall in love with this. I used to use small tweezer to pluck out my leg's hair, in order to avoid the razor, it took me a few hours to do that but this super electronic tweezer works wonder! You won't see the black roots and I have smooth and silkly legs in 20 mins. It does not hurt at all, if you use it on your legs. But, if you use it at your under arm, it won't be a pleasant feeling, of course! I will never use anything else besides this for hair removing anymore! highly recommended!",
-      "id": "79"
-    },
-    "80": {
-      "label": false,
-      "text": "This is not a toy: I guess I was expecting more out of these leave window decals. I just didn't find them attractive after placing them on my window, they seem very cheap, I guess because they are cheap.I threw them away.",
-      "id": "80"
-    },
-    "81": {
-      "label": true,
-      "text": "Wonderful book for anyone running a professional hatchery: This book is aimed more for hatcheries that are raising Trout, Salmon, Catfish and other food fishes. However, there is so much information in this book that even ornamental fish hatcheries will find an incredible amount of useful information. The chapters on Fish Nutrition are especially helpful.",
-      "id": "81"
-    },
-    "82": {
-      "label": true,
-      "text": "Amazing book!!: Once again, Eric Victorino's artistic talent is put into this great free-verse poetry book. I couldn't put it down and I finished it the day I received it in the mail. All of the poems are awesome but the one I found the most interesting was \"It's A People Business.\" All of the experiences in his life, personally and with his band, come to life in this book. Please check it out! It's worth every penny!!",
-      "id": "82"
-    },
-    "83": {
-      "label": true,
-      "text": "The white trumpet contender respect Miles Davis!: The story of the Jazz in the Fifties certainly would be remain unfinished without the ominous presence of this outstanding virtuoso. Baker sound still possesses this alluring hook, this magnetic engagement charm, eloquent expressiveness, enrapturing lyricism and contagious rhythm, despite the elapsed time, which confirms by itself the status of his musicianship.This selection is jus a little sample of the broad universe of his genius. A well thought selection of great musical successes, available, preserved and immortalized by the Digital Technology for our future enjoyment.Absolutely indispensable in your treasured collection.",
-      "id": "83"
-    },
-    "84": {
-      "label": false,
-      "text": "What the?: I'm sorry, maybe it's just me but I can't helping stating that this has to be one of the wrost movies I've seen in my life!Can you say boring? Can you say doesn't make sense at all? The first 30 minutes of the movie were O.K. But it went downhill after that. This movie is a prime example of a director attempting to make a deep movie with a meaningful lesson but failed on all levels. I don't recommend this movie unless you want to go to sleep or you don't have anything else to do.",
-      "id": "84"
-    },
-    "85": {
-      "label": true,
-      "text": "very very good!!!!: linda blair is a young girl who is possessed. and her mother doesn't know what to do until one day when she hears her daughter screaming and stabbind herself she knows what to do GET AN EXORCIZIM!!!",
-      "id": "85"
-    },
-    "86": {
-      "label": true,
-      "text": "Awesome product for the price!: This range extender works as advertised! I am very happy with the purchase. I was a little worried after reading some of the horror stories here, but I have to say, Chovy's review instructions (on this site) were just this ticket to get the repeater up and running in less than 30 minutes. It was unbelievably easy to install! Do not be frightened by negative reviews. If you can set up a wireless network, you can set up this repeater. However, I did upgrade the firmware before I did anything else and maybe that helped. I got the firmware update from the Belkin site.",
-      "id": "86"
-    },
-    "87": {
-      "label": false,
-      "text": "Slight: This book is either a heavily illustrated short story collection or a text-heavy comic. Its unusual format is its most original feature. Its plots are negligible, but its illustrations and text evoke a unique atmosphere of self-conscious nonconformism. Although its target audience is dare-to-be-different teens and college students, its interesting turns of phrase and expressive line drawings are not devoid of interest for general audences.",
-      "id": "87"
-    },
-    "88": {
-      "label": true,
-      "text": "ANgeleyes: Seem to dry up their eyes fairly well, although I haven't seen the color (brown stain) change much yet.",
-      "id": "88"
-    },
-    "89": {
-      "label": false,
-      "text": "Nice Try: Salt Lake 2002 is not a bad game, but it isn't good either. The graphics are excellent, but some of the events are bad. Bobsleigh, and skiing aren't bad but the others are. You dont stay into it for long. I liked it for a while, but it gets boring.",
-      "id": "89"
-    },
-    "90": {
-      "label": false,
-      "text": "Cutler's share of the pie: This book was a major disappointment. I am familiar with books written solely by the Dalai Lama, such as the \"Library of Tibet\" series, which are much more engrossing and have much more substance than Cutler's book. Cutler attempts (successfully, sadly) to have his share of the profitable market that involves the Dalai Lama's writings. The book is insipid, does not try to explain any important issue in the light of Buddhist philosophy, and only rehashes issues that several other westerners already wrote about. It's another big ego trip: we keep hearing time and again about his opportunities to be with the Dalai Lama. What a shame, Cutler. I sold the book as soon as I finished it.",
-      "id": "90"
-    },
-    "91": {
-      "label": false,
-      "text": "Mostly tedious, with interesting parts: I found the writing interesting, and the subject fascinating, but I found myself frustrated by the author's difficulty in talking directly about the status of Muslim women with her interview subjects. The author spent many pages writing about the menus and dress of the many middle and upper-middle class women she interviewed. It seemed as though her interview subjects resisted her efforts to discuss the status of women in their countries, so we too as readers had to wade through much distracting material and misunderstandings about feminism and gender. Great travel stories, but not a great source of information about Muslim women.",
-      "id": "91"
-    },
-    "92": {
-      "label": false,
-      "text": "Sesame Street Toddler: I did not find this game to be as educationally sound as I would expect from Sesame street. There is too much talking before the program will react to a command. The graphics are jerky and the cursor acts like the target is magnetically charged and keeps pushing away the cursor. When the child actually does manage to click on a target, the cursor may still fly to another target and the child is told that his answer is wrong. Another example of educational problems is the pronunciation of \"eggs\" using a long \"a\" sound instead of a short \"e.\" This is not very helpful in teaching a child the sound for short \"e.\" Children that are used to playing computer games by themselves may find that this game is too frustrating to do alone. The open ended learning curve is a great idea. I just wish Sesame Street would hire a truly qualified literacy expert to help clean up the many problems in this program.",
-      "id": "92"
-    },
-    "93": {
-      "label": false,
-      "text": "needs a buzz cut and a point: I avoided reading this book, not because of the hermaphrodite subject matter, but because I have never read a multigenerational family saga that I liked. Many books let me down in the middle, and this was no exception. The beginning of the book was incredible and harrowing, with momentum and characterization. The post-America nextgens part of the saga was so boring I found myself flipping and flipping - always a bad sign. If there was some kind of larger point to all of that, then I must have missed it. Yes there's the identity duality and trinity themes playing out here: man/woman, greek/turkish/american modern/old world sick/healthy innocent/guilty original/reinvented. But it was almost as if the author was saying - here it is again - get it? I like my fiction much more subtle than this.",
-      "id": "93"
-    },
-    "94": {
-      "label": false,
-      "text": "OMG! DO NOT BUY!: I normally don't take the time to submit a review.In this case however, I feel obligated to do so.This is by far one of the worst purchases I have ever made.Here's why.....The contraption is far too bulky.The case's enclosing is unbearable, takes a good minute or so to open it.The texture of the material feels like a cheap toy.The overall design is horrible, something I could make in my basement.For the love of everything sacred, do not buy this thing.",
-      "id": "94"
-    },
-    "95": {
-      "label": true,
-      "text": "Good price, good quality: Comparable HDMI cables can be bought for 45 or more. Even though the price is cheap the quality is good, no problems so far.",
-      "id": "95"
-    },
-    "96": {
-      "label": true,
-      "text": "Good rock music: This is what i call rock music good beat and good lyrics, don't listen to the other reviews. This cd is one of the best, listen to a few songs and you will get hooked. I recommend this cd its awesome.",
-      "id": "96"
-    },
-    "97": {
-      "label": false,
-      "text": "BORING!: This movie is soo boring. How in the hell did this movie make so much at the box office. Do people really want to pay for crappy movies like this. bottom line this is a chick flick nothing is good. And now they are re-releasing this movie with more boring stuff. This is the worst movie ever.",
-      "id": "97"
-    },
-    "98": {
-      "label": false,
-      "text": "Already Rusting: Inferior quality. The plating is thin and rust is coming through the finish. Inexcusable for a product that is designed for use in a humid environment.",
-      "id": "98"
-    },
-    "99": {
-      "label": false,
-      "text": "confusing internet setup: i wanted a camera that could email photos but this camera will not go out through the router and the manual setup , to punch a hole thru router is confusing.",
-      "id": "99"
-    },
-    "55066581ad334ef5844c6f7707525010": {
-      "label": true,
-      "text": "Thought this was super cool, and a really important step in all the physical books' preservation.",
-      "id": "55066581ad334ef5844c6f7707525010"
-    },
-    "fef14d13366f482d9f4e0726b357f178": {
-      "label": true,
-      "text": "There are some amazing hikes around Mt. Fuji.",
-      "id": "fef14d13366f482d9f4e0726b357f178"
-    },
-    "70aed7369aa74031a06f5f3155476d7c": {
-      "label": true,
-      "text": "Thought this was super cool, and a really important step in preserving all the physical books.",
-      "id": "70aed7369aa74031a06f5f3155476d7c"
-    },
-    "ac65d14b710648b8bf3c2a53caf6ac91": {
-      "label": false,
-      "text": "The profits of the business that was most successful were still negative.",
-      "id": "ac65d14b710648b8bf3c2a53caf6ac91"
-    },
-    "ce00e6b1547444259a13c55654e66500": {
-      "label": true,
-      "text": "love them best, they reconnect in hysterically funny and emotionally significant ways.",
-      "id": "ce00e6b1547444259a13c55654e66500"
-    },
-    "8943a94d205b43ceb4420d5ab9c5611a": {
-      "label": true,
-      "text": "Walt Disney's timeless masterpiece is an extravaganza of sight and sound! See the music come to life, hear the pictures burst into song and experience the excitement that is Fantasia over and over again.",
-      "id": "8943a94d205b43ceb4420d5ab9c5611a"
-    },
-    "6af8fc3dd30d4f8caf5a2929fc88534b": {
-      "label": false,
-      "text": "A director struggles with a difficult sex scene between a young actor and actress who can't stand one another. Aided by her loyal assistant, she is hell-bent on getting the scene right without compromise.",
-      "id": "6af8fc3dd30d4f8caf5a2929fc88534b"
-    },
-    "dbe571ed810d40f48170147dcab1c90f": {
-      "label": false,
-      "text": "sound created by drawing directly on the soundtrack).",
-      "id": "dbe571ed810d40f48170147dcab1c90f"
-    },
-    "682102dfc5494f03926d16ae947a6250": {
-      "label": true,
-      "text": "one of glowing admiration! Written by Mark Toscano",
-      "id": "682102dfc5494f03926d16ae947a6250"
-    },
-    "9b044458bb0e4bd68359e62d5fb4b979": {
-      "label": false,
-      "text": "Seth McArdle (Samuel Davis) is a high school senior with an especially full plate. Not only must he navigate the usual social and academic pitfalls of high school, but he has to contend with his young twin sisters, serving as de facto parent in the absence of his deceased mother and deadbeat father. The pressure mounts when the bank calls with a foreclosure warning, and Seth's frustrations spill",
-      "id": "9b044458bb0e4bd68359e62d5fb4b979"
-    },
-    "abf2d24c7d8845769b7368be28f2c25d": {
-      "label": true,
-      "text": "Bjork is a beautiful creature and her music is stellar to anything I've ever heard. This DVD is essential for all Bjork fans, because you find something new every time you watch it.",
-      "id": "abf2d24c7d8845769b7368be28f2c25d"
-    }
-  },
-  "version": 11,
-  "description": "Positive sentiment"
-}

data/concept/lilac/profanity/openai.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a97028bbb8a75913874b83c768c5cdc8ad9ec00aba1ce4296429dd7326165ad7
-size 3247822

data/datasets/local/spotify/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:32224657332b09187a737c73ab634f9d14c9ba9a240bd105f1b9819cde2afcef
-size 37128682

data/datasets/local/spotify/manifest.json

@@ -1,27 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "artist": {
-        "dtype": "string"
-      },
-      "song": {
-        "dtype": "string"
-      },
-      "link": {
-        "dtype": "string"
-      },
-      "text": {
-        "dtype": "string"
-      },
-      "__line_number__": {
-        "dtype": "int64"
-      },
-      "__rowid__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/local/spotify/settings.json

@@ -1 +0,0 @@
-{"ui": {"media_paths": [["text"]]}}

data/datasets/local/spotify/text/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0f1555427c8dc3b2f1e9310f5e71b46297e607f710365e107c73c894d5a8e1b0
-size 2033407

data/datasets/local/spotify/text/lang_detection/signal_manifest.json

@@ -1,36 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "lang_detection(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "lang_detection": {
-            "repeated_field": {
-              "fields": {
-                "lang_code": {
-                  "dtype": "string"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "lang_detection"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/local/spotify/text/sbert/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9796beb630cc3503f3c2ac9db8f71e4c1604570836d78bbf364e801cd427c39e
-size 2709987

data/datasets/local/spotify/text/sbert/embedding/local/outerspace/v34/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d1ba0fe68cc02849b0a20b7f72047c8e9cb8e5ef5b57b0cd642fa0b0be8a6e06
-size 3340135

data/datasets/local/spotify/text/sbert/embedding/local/outerspace/v34/signal_manifest.json

@@ -1,64 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "local/outerspace/v34(text.sbert.*.embedding)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "sbert": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "fields": {
-                    "local/outerspace/v34": {
-                      "dtype": "float32",
-                      "signal": {
-                        "signal_name": "concept_score",
-                        "embedding": "sbert",
-                        "namespace": "local",
-                        "concept_name": "outerspace",
-                        "draft": "main",
-                        "num_negative_examples": 100
-                      },
-                      "bins": [
-                        [
-                          "Not in concept",
-                          null,
-                          0.5
-                        ],
-                        [
-                          "In concept",
-                          0.5,
-                          null
-                        ]
-                      ]
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "concept_score",
-    "embedding": "sbert",
-    "namespace": "local",
-    "concept_name": "outerspace",
-    "draft": "main",
-    "num_negative_examples": 100
-  },
-  "enriched_path": [
-    "text",
-    "sbert",
-    "*",
-    "embedding"
-  ]
-}

data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.keys.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d5df43291782b8c731d4ce56537946654c642a01dc9a4e37de394836362f6b45
-size 3727400

data/datasets/local/spotify/text/sbert/embeddings-00000-of-00001.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:94e10c23d7229541e1f60b791a659d13673b10a03649abf0ae092e0e18c5aee3
-size 170446976

data/datasets/local/spotify/text/sbert/signal_manifest.json

@@ -1,37 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "sbert(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "sbert": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "sbert"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "sbert"
-  },
-  "enriched_path": [
-    "text"
-  ],
-  "embedding_filename_prefix": "embeddings-00000-of-00001"
-}

lilac/concepts/concept.py

@@ -322,7 +322,8 @@ class ConceptModel:
   def _calibrate_on_dataset(self, column_info: ConceptColumnInfo) -> None:
     """Calibrate the model on the embeddings in the provided vector store."""
     db = get_dataset(column_info.namespace, column_info.name)
-    vector_store = db.get_vector_store(self.embedding_name, normalize_path(column_info.path))
+    vector_index = db.get_vector_db_index(self.embedding_name, normalize_path(column_info.path))
+    vector_store = vector_index.get_vector_store()
     keys = vector_store.keys()
     num_samples = min(column_info.num_negative_examples, len(keys))
     sample_keys = random.sample(keys, num_samples)
@@ -354,12 +355,10 @@ class ConceptModel:
 
       item_result: list[Item] = []
       for embedding_item, score in zip(item, scores):
-        item_result.append(
-          lilac_span(
-            start=embedding_item[VALUE_KEY][TEXT_SPAN_START_FEATURE],
-            end=embedding_item[VALUE_KEY][TEXT_SPAN_END_FEATURE],
-            metadata={f'{self.namespace}/{self.concept_name}': score}))
-      result_items.append({self.embedding_name: item_result})
+        span = embedding_item[VALUE_KEY]
+        start, end = span[TEXT_SPAN_START_FEATURE], span[TEXT_SPAN_END_FEATURE]
+        item_result.append(lilac_span(start, end, {'score': score}))
+      result_items.append(item_result)
     return result_items
 
   def coef(self, draft: DraftId) -> np.ndarray:

lilac/config.py

@@ -2,7 +2,7 @@
 import os
 from typing import Any, Literal, Optional, Union, cast
 
-from dotenv import dotenv_values
+from dotenv import load_dotenv
 
 EnvironmentKeys = Union[Literal['LILAC_DATA_PATH'],
                         # Authentication on the demo.
@@ -20,41 +20,44 @@ EnvironmentKeys = Union[Literal['LILAC_DATA_PATH'],
                         Literal['DUCKDB_USE_VIEWS'],
                         # Debugging
                         Literal['DEBUG'], Literal['DISABLE_LOGS']]
-_ENV: Optional[dict[str, Optional[str]]] = None
+
+
+def _init_env() -> None:
+  in_test = os.environ.get('LILAC_TEST', None)
+  # Load the .env files into the environment in order of highest to lowest priority.
+
+  if not in_test:  # Skip local environment variables when testing.
+    load_dotenv('.env.local')
+  load_dotenv('.env.demo')
+  load_dotenv('.env')
+
+  if os.environ.get('LILAC_AUTH_ENABLED', None):
+    if not os.environ.get('GOOGLE_CLIENT_ID', None) or not os.environ.get(
+        'GOOGLE_CLIENT_SECRET', None):
+      raise ValueError(
+        'Missing `GOOGLE_CLIENT_ID` or `GOOGLE_CLIENT_SECRET` when `LILAC_AUTH_ENABLED=true`')
+    SECRET_KEY = os.environ.get('LILAC_OAUTH_SECRET_KEY', None)
+    if not SECRET_KEY:
+      raise ValueError('Missing `LILAC_OAUTH_SECRET_KEY` when `LILAC_AUTH_ENABLED=true`')
+  if os.environ.get('LILAC_AUTH_ENABLED', None):
+    if not os.environ.get('GOOGLE_CLIENT_ID', None) or not os.environ.get(
+        'GOOGLE_CLIENT_SECRET', None):
+      raise ValueError(
+        'Missing `GOOGLE_CLIENT_ID` or `GOOGLE_CLIENT_SECRET` when `LILAC_AUTH_ENABLED=true`')
+    SECRET_KEY = os.environ.get('LILAC_OAUTH_SECRET_KEY', None)
+    if not SECRET_KEY:
+      raise ValueError('Missing `LILAC_OAUTH_SECRET_KEY` when `LILAC_AUTH_ENABLED=true`')
 
 
 def env(key: EnvironmentKeys, default: Optional[Any] = None) -> Any:
   """Return the value of an environment variable."""
-  global _ENV
-  first_load = False
-  # This is done lazily so we can prevent loading local environment variables when testing. The
-  # 'PYTEST_CURRENT_TEST' environment variable is only set after module initialization by pytest.
-
-  if _ENV is None:
-    in_test = os.environ.get('LILAC_TEST', None)
-    _ENV = {
-      **dotenv_values('.env'),  # load shared variables
-      **dotenv_values('.env.demo'),  # load demo-specific environment flags.
-      **(dotenv_values('.env.local') if not in_test else {})
-    }
-    first_load = True
-
-  # Override the file based configs with the current environment, in case flags have changed.
-  environment = {**_ENV, **os.environ}
-
-  if first_load:
-    if environment.get('LILAC_AUTH_ENABLED', None):
-      if not environment.get('GOOGLE_CLIENT_ID', None) or not environment.get(
-          'GOOGLE_CLIENT_SECRET', None):
-        raise ValueError(
-          'Missing `GOOGLE_CLIENT_ID` or `GOOGLE_CLIENT_SECRET` when `LILAC_AUTH_ENABLED=true`')
-      SECRET_KEY = environment.get('LILAC_OAUTH_SECRET_KEY', None)
-      if not SECRET_KEY:
-        raise ValueError('Missing `LILAC_OAUTH_SECRET_KEY` when `LILAC_AUTH_ENABLED=true`')
-
-  return environment.get(key, default)
+  return os.environ.get(key, default)
 
 
 def data_path() -> str:
   """Return the base path for data."""
   return cast(str, env('LILAC_DATA_PATH', './data'))
+
+
+# Initialize the environment at import time.
+_init_env()

lilac/data/dataset.py

@@ -12,7 +12,7 @@ from pydantic import Field as PydanticField
 from pydantic import StrictBool, StrictBytes, StrictFloat, StrictInt, StrictStr, validator
 
 from ..auth import UserInfo
-from ..embeddings.vector_store import VectorStore
+from ..embeddings.vector_store import VectorDBIndex
 from ..schema import VALUE_KEY, Bin, DataType, Path, PathTuple, Schema, normalize_path
 from ..signals.signal import Signal, TextEmbeddingSignal, get_signal_by_type, resolve_signal
 from ..tasks import TaskStepId
@@ -165,7 +165,8 @@ ColumnId = Union[Path, Column]
 
 class DatasetUISettings(BaseModel):
   """The UI persistent settings for a dataset."""
-  media_paths: list[PathTuple] = []
+  media_paths: set[PathTuple] = set()
+  markdown_paths: set[PathTuple] = set()
 
 
 class DatasetSettings(BaseModel):
@@ -282,9 +283,8 @@ class Dataset(abc.ABC):
     pass
 
   @abc.abstractmethod
-  def get_vector_store(self, embedding: str, path: PathTuple) -> VectorStore:
-    # TODO: Instead of this, allow selecting vectors via select_rows.
-    """Get the vector store for a column."""
+  def get_vector_db_index(self, embedding: str, path: PathTuple) -> VectorDBIndex:
+    """Get the vector index for a path and an embedding."""
     pass
 
   @abc.abstractmethod
@@ -462,9 +462,9 @@ def default_settings(dataset: Dataset) -> DatasetSettings:
   stats: list[StatsResult] = list(pool.map(lambda leaf: dataset.stats(leaf), leaf_paths))
   sorted_stats = sorted([stat for stat in stats if stat.avg_text_length],
                         key=lambda stat: stat.avg_text_length or -1.0)
-  media_paths = []
+  media_paths: set[PathTuple] = set()
   if sorted_stats:
-    media_paths = [sorted_stats[-1].path]
+    media_paths = set([sorted_stats[-1].path])
 
   return DatasetSettings(ui=DatasetUISettings(media_paths=media_paths))
 

lilac/data/dataset_duckdb.py

@@ -19,7 +19,7 @@ from typing_extensions import override
 from ..auth import UserInfo
 from ..concepts.concept import ConceptColumnInfo
 from ..config import data_path, env
-from ..embeddings.vector_store import VectorStore
+from ..embeddings.vector_store import VectorDBIndex, VectorStore
 from ..embeddings.vector_store_numpy import NumpyVectorStore
 from ..schema import (
   MANIFEST_FILENAME,
@@ -33,32 +33,31 @@ from ..schema import (
   Field,
   Item,
   Path,
+  PathKey,
   PathTuple,
   RichData,
   Schema,
-  SignalInputType,
   SourceManifest,
-  VectorKey,
   column_paths_match,
   is_float,
   is_integer,
   is_ordinal,
   is_temporal,
   normalize_path,
-  signal_compute_type_supports_dtype,
+  signal_type_supports_dtype,
 )
 from ..signals.concept_labels import ConceptLabelsSignal
 from ..signals.concept_scorer import ConceptScoreSignal
 from ..signals.semantic_similarity import SemanticSimilaritySignal
 from ..signals.signal import (
-  EMBEDDING_KEY,
   Signal,
-  TextEmbeddingModelSignal,
   TextEmbeddingSignal,
+  VectorSignal,
+  get_signal_by_type,
   resolve_signal,
 )
 from ..signals.substring_search import SubstringSignal
-from ..tasks import TaskStepId, TaskStepInfo, progress, set_worker_steps
+from ..tasks import TaskStepId, progress
 from ..utils import DebugTimer, get_dataset_output_dir, log, open_file
 from . import dataset
 from .dataset import (
@@ -95,13 +94,12 @@ from .dataset_utils import (
   flatten,
   flatten_keys,
   merge_schemas,
-  read_embedding_index,
-  replace_embeddings_with_none,
+  read_embeddings_from_disk,
   schema_contains_path,
   sparse_to_dense_compute,
   unflatten,
   wrap_in_dicts,
-  write_item_embeddings_to_disk,
+  write_embeddings_to_disk,
   write_items_to_parquet,
 )
 
@@ -156,8 +154,8 @@ class DatasetDuckDB(Dataset):
     self._signal_manifests: list[SignalManifest] = []
     self.con = duckdb.connect(database=':memory:')
 
-    # Maps a column path and embedding to the vector store. This is lazily generated as needed.
-    self._col_vector_stores: dict[PathTuple, VectorStore] = {}
+    # Maps a path and embedding to the vector index. This is lazily generated as needed.
+    self._vector_indices: dict[tuple[PathKey, str], VectorDBIndex] = {}
     self.vector_store_cls = vector_store_cls
     self._manifest_lock = threading.Lock()
 
@@ -196,7 +194,8 @@ class DatasetDuckDB(Dataset):
           signal_manifest = SignalManifest.parse_raw(f.read())
         self._signal_manifests.append(signal_manifest)
         signal_files = [os.path.join(root, f) for f in signal_manifest.files]
-        self._create_view(signal_manifest.parquet_id, signal_files)
+        if signal_files:
+          self._create_view(signal_manifest.parquet_id, signal_files)
 
     merged_schema = merge_schemas([self._source_manifest.data_schema] +
                                   [m.data_schema for m in self._signal_manifests])
@@ -212,10 +211,13 @@ class DatasetDuckDB(Dataset):
     # NOTE: "root_column" for each signal is defined as the top-level column.
     select_sql = ', '.join([f'{SOURCE_VIEW_NAME}.*'] + [(
       f'{_escape_col_name(manifest.parquet_id)}.{_escape_col_name(_root_column(manifest))} '
-      f'AS {_escape_col_name(manifest.parquet_id)}') for manifest in self._signal_manifests])
+      f'AS {_escape_col_name(manifest.parquet_id)}')
+                                                        for manifest in self._signal_manifests
+                                                        if manifest.files])
     join_sql = ' '.join([SOURCE_VIEW_NAME] + [
       f'join {_escape_col_name(manifest.parquet_id)} using ({UUID_COLUMN},)'
       for manifest in self._signal_manifests
+      if manifest.files
     ])
     view_or_table = 'TABLE'
     use_views = env('DUCKDB_USE_VIEWS', 0) or 0
@@ -267,105 +269,42 @@ class DatasetDuckDB(Dataset):
     raise NotImplementedError('count is not yet implemented for DuckDB.')
 
   @override
-  def get_vector_store(self, embedding: str, path: PathTuple) -> VectorStore:
+  def get_vector_db_index(self, embedding: str, path: PathTuple) -> VectorDBIndex:
     # Refresh the manifest to make sure we have the latest signal manifests.
     self.manifest()
+    index_key = (path, embedding)
+    if index_key in self._vector_indices:
+      return self._vector_indices[index_key]
 
-    if path[-1] != EMBEDDING_KEY:
-      path = (*path, embedding, PATH_WILDCARD, EMBEDDING_KEY)
-
-    if path not in self._col_vector_stores:
-      manifests = [
-        m for m in self._signal_manifests
-        if schema_contains_path(m.data_schema, path) and m.embedding_filename_prefix
-      ]
-      if not manifests:
-        raise ValueError(f'No embedding found for path {path}.')
-      if len(manifests) > 1:
-        raise ValueError(f'Multiple embeddings found for path {path}. Got: {manifests}')
-      manifest = manifests[0]
-      if not manifest.embedding_filename_prefix:
-        raise ValueError(f'Signal manifest for path {path} is not an embedding. '
-                         f'Got signal manifest: {manifest}')
-
-      signal_name = cast(str, manifest.signal.signal_name)
-      filepath_prefix = os.path.join(self.dataset_path, _signal_dir(manifest.enriched_path),
-                                     signal_name, manifest.embedding_filename_prefix)
-      keys, embeddings = read_embedding_index(filepath_prefix)
-      # Get all the embeddings and pass it to the vector store.
-      vector_store = self.vector_store_cls()
-      vector_store.add(keys, embeddings)
-      # Cache the vector store.
-      self._col_vector_stores[path] = vector_store
-
-    return self._col_vector_stores[path]
-
-  def _prepare_signal(
-      self,
-      signal: Signal,
-      source_path: PathTuple,
-      manifest: DatasetManifest,
-      compute_dependencies: Optional[bool] = False,
-      task_step_id: Optional[TaskStepId] = None) -> tuple[PathTuple, Optional[TaskStepId]]:
-    """Run all the signals dependencies required to run this signal.
-
-    Args:
-      signal: The signal to prepare.
-      source_path: The source path the signal is running over.
-      compute_dependencies: If True, signals will get computed for the whole column. If False,
-        throw if the required inputs are not computed yet.
-      task_step_id: The TaskStepId used to run the signal.
-
-    Returns
-      The final path the signal will be run over and the new step id for the final signal.
-    """
-    is_value_path = False
-    if source_path[-1] == VALUE_KEY:
-      is_value_path = True
-      source_path = source_path[:-1]
-
-    new_path = source_path
-
-    signals_to_compute: list[tuple[PathTuple, Signal]] = []
-    if isinstance(signal, TextEmbeddingModelSignal):
-      embedding_signal = signal.get_embedding_signal()
-      new_path = (*new_path, embedding_signal.key(), PATH_WILDCARD, EMBEDDING_KEY)
-      if new_path not in manifest.data_schema.leafs:
-        if not compute_dependencies:
-          raise ValueError(f'Embedding signal "{embedding_signal.key()}" is not computed over '
-                           f'{source_path}. Please run `dataset.compute_signal` over '
-                           f'{source_path} first.')
-        signals_to_compute.append((new_path, embedding_signal))
-
-    new_steps = len(signals_to_compute)
-    # Setup the task steps so the task progress indicator knows the number of steps before they are
-    # computed.
-    task_id: Optional[str] = None
-    step_id: Optional[int] = None
-    if task_step_id:
-      (task_id, step_id) = task_step_id
-      if task_id != '' and new_steps:
-        # Make a step for the parent.
-        set_worker_steps(task_id, [TaskStepInfo()] * (new_steps + 1))
-
-    for i, (new_path, signal) in enumerate(signals_to_compute):
-      if new_path not in manifest.data_schema.leafs:
-        self.compute_signal(
-          signal, source_path, task_step_id=(task_id, i) if task_id is not None else None)
-
-    if is_value_path:
-      new_path = (*new_path, VALUE_KEY)
-
-    new_task_id: Optional[TaskStepId] = None
-    if task_id is not None and step_id is not None:
-      new_task_id = (task_id, step_id + new_steps)
-    return (new_path, new_task_id)
+    manifests = [
+      m for m in self._signal_manifests
+      if schema_contains_path(m.data_schema, path) and m.embedding_filename_prefix
+    ]
+    if not manifests:
+      raise ValueError(f'No embedding found for path {path}.')
+    if len(manifests) > 1:
+      raise ValueError(f'Multiple embeddings found for path {path}. Got: {manifests}')
+    manifest = manifests[0]
+    if not manifest.embedding_filename_prefix:
+      raise ValueError(f'Signal manifest for path {path} is not an embedding. '
+                       f'Got signal manifest: {manifest}')
+
+    signal_name = cast(str, manifest.signal.signal_name)
+    filepath_prefix = os.path.join(self.dataset_path, _signal_dir(manifest.enriched_path),
+                                   signal_name, manifest.embedding_filename_prefix)
+    spans, embeddings = read_embeddings_from_disk(filepath_prefix)
+    vector_index = VectorDBIndex(self.vector_store_cls, spans, embeddings)
+    # Cache the vector index.
+    self._vector_indices[index_key] = vector_index
+    return vector_index
 
   @override
   def compute_signal(self,
                      signal: Signal,
                      leaf_path: Path,
                      task_step_id: Optional[TaskStepId] = None) -> None:
+    if isinstance(signal, TextEmbeddingSignal):
+      return self.compute_embedding(signal.name, leaf_path, task_step_id)
     source_path = normalize_path(leaf_path)
     manifest = self.manifest()
 
@@ -373,10 +312,6 @@ class DatasetDuckDB(Dataset):
       # Make a dummy task step so we report progress via tqdm.
       task_step_id = ('', 0)
 
-    # Prepare the dependencies of this signal.
-    signal_source_path, task_step_id = self._prepare_signal(
-      signal, source_path, manifest, compute_dependencies=True, task_step_id=task_step_id)
-
     # The manifest may have changed after computing the dependencies.
     manifest = self.manifest()
 
@@ -392,9 +327,6 @@ class DatasetDuckDB(Dataset):
     df = select_rows_result.df()
     values = df['value']
 
-    source_path = signal_source_path
-    signal_col.path = source_path
-
     enriched_path = _col_destination_path(signal_col, is_computed_signal=True)
     spec = _split_path_into_subpaths_of_lists(enriched_path)
     output_dir = os.path.join(self.dataset_path, _signal_dir(enriched_path))
@@ -403,20 +335,6 @@ class DatasetDuckDB(Dataset):
     for uuid, item in zip(df[UUID_COLUMN], enriched_signal_items):
       item[UUID_COLUMN] = uuid
 
-    is_embedding = isinstance(signal, TextEmbeddingSignal)
-    embedding_filename_prefix = None
-    if is_embedding:
-      embedding_filename_prefix = os.path.basename(
-        write_item_embeddings_to_disk(
-          keys=df[UUID_COLUMN],
-          embeddings=values,
-          output_dir=output_dir,
-          shard_index=0,
-          num_shards=1))
-
-      # Replace the embeddings with None so they are not serialized in the parquet file.
-      enriched_signal_items = (replace_embeddings_with_none(item) for item in enriched_signal_items)
-
     enriched_signal_items = list(enriched_signal_items)
     parquet_filename, _ = write_items_to_parquet(
       items=enriched_signal_items,
@@ -431,12 +349,55 @@ class DatasetDuckDB(Dataset):
       data_schema=signal_schema,
       signal=signal,
       enriched_path=source_path,
+      parquet_id=make_parquet_id(signal, source_path, is_computed_signal=True))
+    signal_manifest_filepath = os.path.join(output_dir, SIGNAL_MANIFEST_FILENAME)
+    with open_file(signal_manifest_filepath, 'w') as f:
+      f.write(signal_manifest.json(exclude_none=True, indent=2))
+    log(f'Wrote signal output to {output_dir}')
+
+  @override
+  def compute_embedding(self,
+                        embedding: str,
+                        leaf_path: Path,
+                        task_step_id: Optional[TaskStepId] = None) -> None:
+    source_path = normalize_path(leaf_path)
+    manifest = self.manifest()
+
+    if task_step_id is None:
+      # Make a dummy task step so we report progress via tqdm.
+      task_step_id = ('', 0)
+
+    signal = get_signal_by_type(embedding, TextEmbeddingSignal)()
+    signal_col = Column(path=source_path, alias='value', signal_udf=signal)
+    select_rows_result = self.select_rows([signal_col],
+                                          task_step_id=task_step_id,
+                                          resolve_span=True)
+    df = select_rows_result.df()
+    values = df['value']
+
+    enriched_path = _col_destination_path(signal_col, is_computed_signal=True)
+    output_dir = os.path.join(self.dataset_path, _signal_dir(enriched_path))
+    signal_schema = create_signal_schema(signal, source_path, manifest.data_schema)
+    embedding_filename_prefix = os.path.basename(
+      write_embeddings_to_disk(
+        uuids=df[UUID_COLUMN],
+        signal_items=values,
+        output_dir=output_dir,
+        shard_index=0,
+        num_shards=1))
+
+    signal_manifest = SignalManifest(
+      files=[],
+      data_schema=signal_schema,
+      signal=signal,
+      enriched_path=source_path,
       parquet_id=make_parquet_id(signal, source_path, is_computed_signal=True),
       embedding_filename_prefix=embedding_filename_prefix)
     signal_manifest_filepath = os.path.join(output_dir, SIGNAL_MANIFEST_FILENAME)
+
     with open_file(signal_manifest_filepath, 'w') as f:
       f.write(signal_manifest.json(exclude_none=True, indent=2))
-    log(f'Wrote signal output to {output_dir}')
+    log(f'Wrote embedding index to {output_dir}')
 
   @override
   def delete_signal(self, signal_path: Path) -> None:
@@ -493,10 +454,9 @@ class DatasetDuckDB(Dataset):
 
       # Signal transforms must have the same dtype as the leaf field.
       signal = cast(Signal, col.signal_udf)
-      compute_type = signal.compute_type
-      if not signal_compute_type_supports_dtype(compute_type, leaf.dtype):
+      if not signal_type_supports_dtype(signal.input_type, leaf.dtype):
         raise ValueError(f'Leaf "{path}" has dtype "{leaf.dtype}" which is not supported '
-                         f'by "{signal.key()}" with signal input type "{compute_type}".')
+                         f'by "{signal.key()}" with signal input type "{signal.input_type}".')
 
   def _validate_selection(self, columns: Sequence[Column], select_schema: Schema) -> None:
     # Validate all the columns and make sure they exist in the `select_schema`.
@@ -728,8 +688,7 @@ class DatasetDuckDB(Dataset):
     if not udf_cols_to_sort_by:
       return None
     udf_col = udf_cols_to_sort_by[0]
-    if udf_col.signal_udf and (udf_col.signal_udf.compute_type
-                               not in [SignalInputType.TEXT_EMBEDDING]):
+    if udf_col.signal_udf and not isinstance(udf_col.signal_udf, VectorSignal):
       return None
     return udf_col
 
@@ -801,14 +760,6 @@ class DatasetDuckDB(Dataset):
     if (UUID_COLUMN,) not in col_paths:
       cols.append(column_from_identifier(UUID_COLUMN))
 
-    # Prepare UDF columns. Throw an error if they are not computed. Update the paths of the UDFs so
-    # they match the paths of the columns defined by splits and embeddings.
-    for col in cols:
-      if col.signal_udf:
-        # Do not auto-compute dependencies, throw an error if they are not computed.
-        col.path, _ = self._prepare_signal(
-          col.signal_udf, col.path, manifest, compute_dependencies=False)
-
     schema = manifest.data_schema
 
     if combine_columns:
@@ -825,9 +776,8 @@ class DatasetDuckDB(Dataset):
     for udf_col in udf_columns:
       if isinstance(udf_col.signal_udf, ConceptScoreSignal):
         # Set dataset information on the signal.
-        source_path = udf_col.path if udf_col.path[-1] != EMBEDDING_KEY else udf_col.path[:-3]
         udf_col.signal_udf.set_column_info(
-          ConceptColumnInfo(namespace=self.namespace, name=self.dataset_name, path=source_path))
+          ConceptColumnInfo(namespace=self.namespace, name=self.dataset_name, path=udf_col.path))
 
       if isinstance(udf_col.signal_udf, (ConceptScoreSignal, ConceptLabelsSignal)):
         # Concept are access controlled so we tell it about the user.
@@ -863,19 +813,20 @@ class DatasetDuckDB(Dataset):
 
     topk_udf_col = self._topk_udf_to_sort_by(udf_columns, sort_by, limit, sort_order)
     if topk_udf_col:
-      key_prefixes: Optional[Iterable[VectorKey]] = None
+      path_keys: Optional[Iterable[PathKey]] = None
       if where_query:
         # If there are filters, we need to send UUIDs to the top k query.
         df = con.execute(f'SELECT {UUID_COLUMN} FROM t {where_query}').df()
         total_num_rows = len(df)
-        key_prefixes = df[UUID_COLUMN]
+        # Convert UUIDs to path keys.
+        path_keys = [(uuid,) for uuid in df[UUID_COLUMN]]
 
-      topk_signal = cast(TextEmbeddingModelSignal, topk_udf_col.signal_udf)
+      topk_signal = cast(VectorSignal, topk_udf_col.signal_udf)
       # The input is an embedding.
-      vector_store = self.get_vector_store(topk_signal.embedding, topk_udf_col.path)
+      vector_index = self.get_vector_db_index(topk_signal.embedding, topk_udf_col.path)
       k = (limit or 0) + (offset or 0)
-      topk = topk_signal.vector_compute_topk(k, vector_store, key_prefixes)
-      topk_uuids = list(dict.fromkeys([cast(str, key[0]) for key, _ in topk]))
+      topk = topk_signal.vector_compute_topk(k, vector_index, path_keys)
+      topk_uuids = list(dict.fromkeys([cast(str, path_key[0]) for path_key, _ in topk]))
 
       # Ignore all the other filters and filter DuckDB results only by the top k UUIDs.
       uuid_filter = Filter(path=(UUID_COLUMN,), op=ListOp.IN, value=topk_uuids)
@@ -995,13 +946,12 @@ class DatasetDuckDB(Dataset):
       with DebugTimer(f'Computing signal "{signal.signal_name}"'):
         signal.setup()
 
-        if signal.compute_type in [SignalInputType.TEXT_EMBEDDING]:
-          # The input is an embedding.
-          embedding_signal = cast(TextEmbeddingModelSignal, signal)
-          vector_store = self.get_vector_store(embedding_signal.embedding, udf_col.path)
+        if isinstance(signal, VectorSignal):
+          embedding_signal = signal
+          vector_store = self.get_vector_db_index(embedding_signal.embedding, udf_col.path)
           flat_keys = list(flatten_keys(df[UUID_COLUMN], input))
           signal_out = sparse_to_dense_compute(
-            iter(flat_keys), lambda keys: signal.vector_compute(keys, vector_store))
+            iter(flat_keys), lambda keys: embedding_signal.vector_compute(keys, vector_store))
           # Add progress.
           if task_step_id is not None:
             signal_out = progress(
@@ -1115,14 +1065,6 @@ class DatasetDuckDB(Dataset):
     if (UUID_COLUMN,) not in col_paths:
       cols.append(column_from_identifier(UUID_COLUMN))
 
-    # Prepare UDF columns. Throw an error if they are not computed. Update the paths of the UDFs so
-    # they match the paths of the columns defined by splits and embeddings.
-    for col in cols:
-      if col.signal_udf:
-        # Do not auto-compute dependencies, throw an error if they are not computed.
-        col.path, _ = self._prepare_signal(
-          col.signal_udf, col.path, manifest, compute_dependencies=False)
-
     self._normalize_searches(searches, manifest)
     search_udfs = self._search_udfs(searches, manifest)
     cols.extend([search_udf.udf for search_udf in search_udfs])
@@ -1188,6 +1130,8 @@ class DatasetDuckDB(Dataset):
     select_leaf = select_leaf or column.signal_udf is not None
 
     for m in parquet_manifests:
+      if not m.files:
+        continue
       # Skip this parquet file if it doesn't contain the path.
       if not schema_contains_path(m.data_schema, path):
         continue
@@ -1284,9 +1228,8 @@ class DatasetDuckDB(Dataset):
         if not embedding:
           raise ValueError(f'Please provide an embedding for semantic search. Got search: {search}')
 
-        embedding_path = (*search_path, embedding, PATH_WILDCARD, EMBEDDING_KEY)
         try:
-          manifest.data_schema.get_field(embedding_path)
+          manifest.data_schema.get_field((*search_path, embedding))
         except Exception as e:
           raise ValueError(
             f'Embedding {embedding} has not been computed. '
@@ -1314,7 +1257,7 @@ class DatasetDuckDB(Dataset):
               output_path=_col_destination_path(concept_labels_udf),
               sort=None))
 
-        udf = Column(path=embedding_path, signal_udf=search_signal)
+        udf = Column(path=search_path, signal_udf=search_signal)
 
         output_path = _col_destination_path(udf)
         search_udfs.append(
@@ -1373,7 +1316,7 @@ class DatasetDuckDB(Dataset):
         sql_op = BINARY_OP_TO_SQL[cast(BinaryOp, f.op)]
         filter_val = cast(FeatureValue, f.value)
         if isinstance(filter_val, str):
-          filter_val = f"'{filter_val}'"
+          filter_val = _escape_string_literal(filter_val)
         elif isinstance(filter_val, bytes):
           filter_val = _bytes_to_blob_literal(filter_val)
         else:

lilac/data/dataset_test_utils.py

@@ -4,8 +4,10 @@ import pathlib
 from datetime import datetime
 from typing import Optional, Type, cast
 
+import numpy as np
 from typing_extensions import Protocol
 
+from ..embeddings.vector_store import VectorDBIndex, VectorStore
 from ..schema import (
   MANIFEST_FILENAME,
   PARQUET_FILENAME_PREFIX,
@@ -13,14 +15,13 @@ from ..schema import (
   DataType,
   Field,
   Item,
+  PathKey,
   Schema,
   SourceManifest,
-  field,
 )
-from ..signals.signal import EMBEDDING_KEY
 from ..utils import get_dataset_output_dir, open_file
 from .dataset import Dataset
-from .dataset_utils import is_primitive, lilac_span, write_items_to_parquet
+from .dataset_utils import is_primitive, write_items_to_parquet
 
 TEST_NAMESPACE = 'test_namespace'
 TEST_DATASET_NAME = 'test_dataset'
@@ -109,11 +110,16 @@ def enriched_item(value: Optional[Item] = None, metadata: dict[str, Item] = {})
   return {VALUE_KEY: value, **metadata}
 
 
-def enriched_embedding_span(start: int, end: int, metadata: dict[str, Item] = {}) -> Item:
-  """Makes an item that represents an embedding span that was enriched with metadata."""
-  return lilac_span(start, end, {EMBEDDING_KEY: {VALUE_KEY: None, **metadata}})
+def make_vector_index(vector_store_cls: Type[VectorStore],
+                      vector_dict: dict[PathKey, list[list[float]]]) -> VectorDBIndex:
+  """Make a vector index from a dictionary of vector keys to vectors."""
+  embeddings: list[np.ndarray] = []
+  spans: list[tuple[PathKey, list[tuple[int, int]]]] = []
+  for path_key, vectors in vector_dict.items():
+    vector_spans: list[tuple[int, int]] = []
+    for i, vector in enumerate(vectors):
+      embeddings.append(np.array(vector))
+      vector_spans.append((0, 0))
+    spans.append((path_key, vector_spans))
 
-
-def enriched_embedding_span_field(metadata: Optional[object] = {}) -> Field:
-  """Makes a field that represents an embedding span that was enriched with metadata."""
-  return field('string_span', fields={EMBEDDING_KEY: field('embedding', fields=metadata)})
+  return VectorDBIndex(vector_store_cls, spans, np.array(embeddings))

lilac/data/dataset_utils.py

@@ -22,6 +22,7 @@ from ..schema import (
   VALUE_KEY,
   Field,
   Item,
+  PathKey,
   PathTuple,
   Schema,
   VectorKey,
@@ -33,6 +34,7 @@ from ..signals.signal import EMBEDDING_KEY, Signal
 from ..utils import file_exists, log, open_file
 
 _KEYS_SUFFIX = '.keys.pkl'
+_SPANS_SUFFIX = '.spans.pkl'
 _EMBEDDINGS_SUFFIX = '.npy'
 
 
@@ -218,53 +220,58 @@ def create_signal_schema(signal: Signal, source_path: PathTuple, current_schema:
   return schema({UUID_COLUMN: 'string', **cast(dict, enriched_schema.fields)})
 
 
-def write_item_embeddings_to_disk(keys: Iterable[str], embeddings: Iterable[Item], output_dir: str,
-                                  shard_index: int, num_shards: int) -> str:
+def write_embeddings_to_disk(uuids: Iterable[str], signal_items: Iterable[Item], output_dir: str,
+                             shard_index: int, num_shards: int) -> str:
   """Write a set of embeddings to disk."""
   output_path_prefix = embedding_index_filename_prefix(output_dir, shard_index, num_shards)
 
-  # Restrict the keys to only those that are embeddings.
+  # For each item, we have a list of embedding spans.
   def embedding_predicate(input: Any) -> bool:
-    return isinstance(input, np.ndarray)
+    return (isinstance(input, list) and len(input) > 0 and isinstance(input[0], dict) and
+            EMBEDDING_KEY in input[0])
 
-  flat_keys = flatten_keys(keys, embeddings, is_primitive_predicate=embedding_predicate)
-  flat_embeddings = cast(Iterable[Item],
-                         flatten(embeddings, is_primitive_predicate=embedding_predicate))
+  path_keys = flatten_keys(uuids, signal_items, is_primitive_predicate=embedding_predicate)
+  all_embeddings = cast(Iterable[Item],
+                        flatten(signal_items, is_primitive_predicate=embedding_predicate))
 
   embedding_vectors: list[np.ndarray] = []
-  embedding_keys: list[VectorKey] = []
-  for key, lilac_embedding in zip(flat_keys, flat_embeddings):
-    if not key or not lilac_embedding or EMBEDDING_KEY not in lilac_embedding:
+  all_spans: list[tuple[PathKey, list[tuple[int, int]]]] = []
+  for path_key, embeddings in zip(path_keys, all_embeddings):
+    if not path_key or not embeddings:
       # Sparse embeddings may not have an embedding for every key.
       continue
 
-    # We use squeeze here because embedding functions can return outer dimensions of 1.
-    embedding_vectors.append(lilac_embedding[EMBEDDING_KEY].reshape(-1))
-    embedding_keys.append(key)
+    spans: list[tuple[int, int]] = []
+    for e in embeddings:
+      span = e[VALUE_KEY]
+      vector = e[EMBEDDING_KEY]
+      # We squeeze here because embedding functions can return outer dimensions of 1.
+      embedding_vectors.append(vector.reshape(-1))
+      spans.append((span[TEXT_SPAN_START_FEATURE], span[TEXT_SPAN_END_FEATURE]))
+    all_spans.append((path_key, spans))
 
   embedding_matrix = np.array(embedding_vectors)
-
   # Write the embedding index and the ordered UUID column to disk so they can be joined later.
 
   with open_file(output_path_prefix + _EMBEDDINGS_SUFFIX, 'wb') as f:
     np.save(cast(str, f), embedding_matrix, allow_pickle=False)
-  with open_file(output_path_prefix + _KEYS_SUFFIX, 'wb') as f:
-    pickle.dump(embedding_keys, f)
+  with open_file(output_path_prefix + _SPANS_SUFFIX, 'wb') as f:
+    pickle.dump(all_spans, f)
 
   return output_path_prefix
 
 
-def read_embedding_index(filepath_prefix: str) -> tuple[list[VectorKey], np.ndarray]:
-  """Reads the embedding index for a column from disk."""
+def read_embeddings_from_disk(
+    filepath_prefix: str) -> tuple[list[tuple[PathKey, list[tuple[int, int]]]], np.ndarray]:
+  """Reads the embeddings from disk."""
   if not file_exists(filepath_prefix + _EMBEDDINGS_SUFFIX):
     raise ValueError(F'Embedding index does not exist at path {filepath_prefix}. '
                      'Please run dataset.compute_signal() on the embedding signal first.')
-
   # Read the embedding index from disk.
   embeddings = np.load(filepath_prefix + _EMBEDDINGS_SUFFIX, allow_pickle=False)
-  with open_file(filepath_prefix + _KEYS_SUFFIX, 'rb') as f:
-    index_keys: list[VectorKey] = pickle.load(f)
-  return index_keys, embeddings
+  with open_file(filepath_prefix + _SPANS_SUFFIX, 'rb') as f:
+    spans: list[tuple[PathKey, list[tuple[int, int]]]] = pickle.load(f)
+  return spans, embeddings
 
 
 def write_items_to_parquet(items: Iterable[Item], output_dir: str, schema: Schema,

lilac/embeddings/vector_store.py

@@ -1,9 +1,10 @@
 """Interface for storing vectors."""
 
 import abc
-from typing import Iterable, Optional
+from typing import Iterable, Optional, Type
 
 import numpy as np
+from typing_extensions import TypedDict
 
 from ..schema import VectorKey
 
@@ -29,7 +30,7 @@ class VectorStore(abc.ABC):
     pass
 
   @abc.abstractmethod
-  def get(self, keys: Iterable[VectorKey]) -> np.ndarray:
+  def get(self, keys: Optional[Iterable[VectorKey]] = None) -> np.ndarray:
     """Return the embeddings for given keys.
 
     Args:
@@ -43,15 +44,95 @@ class VectorStore(abc.ABC):
   def topk(self,
            query: np.ndarray,
            k: int,
-           key_prefixes: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, float]]:
+           keys: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, float]]:
     """Return the top k most similar vectors.
 
     Args:
       query: The query vector.
       k: The number of results to return.
-      key_prefixes: Optional key prefixes to restrict the search to.
+      keys: Optional keys to restrict the search to.
 
     Returns
       A list of (key, score) tuples.
     """
     raise NotImplementedError
+
+
+class SpanVector(TypedDict):
+  """A span with a vector."""
+  span: tuple[int, int]
+  vector: np.ndarray
+
+
+PathKey = VectorKey
+
+
+class VectorDBIndex:
+  """Stores and retrives span vectors.
+
+  This wraps a regular vector store by adding a mapping from path keys, such as (uuid1, 0),
+  to span keys, such as (uuid1, 0, 0), which denotes the first span in the (uuid1, 0) text document.
+  """
+
+  def __init__(self, vector_store_cls: Type[VectorStore],
+               spans: list[tuple[PathKey, list[tuple[int, int]]]], embeddings: np.ndarray) -> None:
+    vector_keys = [(*path_key, i) for path_key, spans in spans for i in range(len(spans))]
+    self._vector_store = vector_store_cls()
+    self._vector_store.add(vector_keys, embeddings)
+    # Map a path key to spans for that path.
+    self._id_to_spans: dict[PathKey, list[tuple[int, int]]] = {}
+    self._id_to_spans.update(spans)
+
+  def get_vector_store(self) -> VectorStore:
+    """Return the vector store."""
+    return self._vector_store
+
+  def get(self, keys: Iterable[PathKey]) -> Iterable[list[SpanVector]]:
+    """Return the spans with vectors for each key in `keys`.
+
+    Args:
+      keys: The keys to return the vectors for.
+
+    Returns
+      The span vectors for the given keys.
+    """
+    all_spans: list[list[tuple[int, int]]] = []
+    vector_keys: list[VectorKey] = []
+    for path_key in keys:
+      spans = self._id_to_spans[path_key]
+      all_spans.append(spans)
+      vector_keys.extend([(*path_key, i) for i in range(len(spans))])
+
+    all_vectors = self._vector_store.get(vector_keys)
+    offset = 0
+    for spans in all_spans:
+      vectors = all_vectors[offset:offset + len(spans)]
+      yield [{'span': span, 'vector': vector} for span, vector in zip(spans, vectors)]
+      offset += len(spans)
+
+  def topk(self,
+           query: np.ndarray,
+           k: int,
+           path_keys: Optional[Iterable[PathKey]] = None) -> list[tuple[PathKey, float]]:
+    """Return the top k most similar vectors.
+
+    Args:
+      query: The query vector.
+      k: The number of results to return.
+      path_keys: Optional key prefixes to restrict the search to.
+
+    Returns
+      A list of (key, score) tuples.
+    """
+    vector_keys: Optional[list[VectorKey]] = None
+    if path_keys:
+      vector_keys = [
+        (*path_key, i) for path_key in path_keys for i in range(len(self._id_to_spans[path_key]))
+      ]
+    vector_key_scores = self._vector_store.topk(query, k, vector_keys)
+    path_key_scores: dict[PathKey, float] = {}
+    for (*path_key_list, _), score in vector_key_scores:
+      path_key = tuple(path_key_list)
+      if path_key not in path_key_scores:
+        path_key_scores[path_key] = score
+    return list(path_key_scores.items())

lilac/embeddings/vector_store_numpy.py

@@ -34,13 +34,11 @@ class NumpyVectorStore(VectorStore):
     # Cast to float32 since dot product with float32 is 40-50x faster than float16 and 2.5x faster
     # than float64.
     self._embeddings = embeddings.astype(np.float32)
-
-    index = pd.MultiIndex.from_tuples(keys)
     row_indices = np.arange(len(self._embeddings), dtype=np.uint32)
-    self._lookup = pd.Series(row_indices, index=index)
+    self._lookup = pd.Series(row_indices, index=keys)
 
   @override
-  def get(self, keys: Iterable[VectorKey]) -> np.ndarray:
+  def get(self, keys: Optional[Iterable[VectorKey]] = None) -> np.ndarray:
     """Return the embeddings for given keys.
 
     Args:
@@ -49,6 +47,8 @@ class NumpyVectorStore(VectorStore):
     Returns
       The embeddings for the given keys.
     """
+    if not keys:
+      return self._embeddings
     locs = self._lookup.loc[cast(list[str], keys)]
     return self._embeddings.take(locs, axis=0)
 
@@ -56,14 +56,10 @@ class NumpyVectorStore(VectorStore):
   def topk(self,
            query: np.ndarray,
            k: int,
-           key_prefixes: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, float]]:
-    if key_prefixes is not None:
-      # Cast tuples of length 1 to the element itself to avoid a pandas bug.
-      key_prefixes = cast(
-        list[VectorKey],
-        [k[0] if isinstance(k, tuple) and len(k) == 1 else k for k in key_prefixes])
+           keys: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, float]]:
+    if keys is not None:
       # This uses the hierarchical index (MutliIndex) to do a prefix lookup.
-      row_indices = self._lookup.loc[cast(list[str], key_prefixes)]
+      row_indices = self._lookup.loc[cast(list[str], keys)]
       keys, embeddings = list(row_indices.index), self._embeddings.take(row_indices, axis=0)
     else:
       keys, embeddings = self._keys, self._embeddings

lilac/router_concept.py

@@ -125,7 +125,7 @@ class ScoreBody(BaseModel):
 
 class ScoreResponse(BaseModel):
   """Response body for the score endpoint."""
-  scores: list[dict]
+  scored_spans: list[list[dict]]
   model_synced: bool
 
 
@@ -212,7 +212,7 @@ def score(namespace: str, concept_name: str, embedding_name: str, body: ScoreBod
   model_updated = DISK_CONCEPT_MODEL_DB.sync(model, user)
   # TODO(smilkov): Support images.
   texts = [example.text or '' for example in body.examples]
-  return ScoreResponse(scores=model.score(body.draft, texts), model_synced=model_updated)
+  return ScoreResponse(scored_spans=model.score(body.draft, texts), model_synced=model_updated)
 
 
 class Examples(OpenAISchema):

lilac/router_dataset.py

@@ -29,13 +29,7 @@ from .schema import Bin, Path, normalize_path
 from .signals.concept_labels import ConceptLabelsSignal
 from .signals.concept_scorer import ConceptScoreSignal
 from .signals.semantic_similarity import SemanticSimilaritySignal
-from .signals.signal import (
-  Signal,
-  TextEmbeddingModelSignal,
-  TextEmbeddingSignal,
-  TextSignal,
-  resolve_signal,
-)
+from .signals.signal import Signal, TextEmbeddingSignal, TextSignal, resolve_signal
 from .signals.substring_search import SubstringSignal
 from .tasks import TaskId, task_manager
 from .utils import DatasetInfo, list_datasets
@@ -176,8 +170,7 @@ class ListFilter(BaseModel):
 Filter = Union[BinaryFilter, UnaryFilter, ListFilter]
 
 AllSignalTypes = Union[ConceptScoreSignal, ConceptLabelsSignal, SubstringSignal,
-                       SemanticSimilaritySignal, TextEmbeddingModelSignal, TextEmbeddingSignal,
-                       TextSignal, Signal]
+                       SemanticSimilaritySignal, TextEmbeddingSignal, TextSignal, Signal]
 
 
 # We override the `Column` class so we can add explicitly all signal types for better OpenAPI spec.

lilac/schema.py

@@ -40,6 +40,7 @@ PathKeyedItem = tuple[Path, Item]
 # These fields are for for python only and not written to a schema.
 RichData = Union[str, bytes]
 VectorKey = tuple[Union[StrictStr, StrictInt], ...]
+PathKey = VectorKey
 
 
 class DataType(str, Enum):
@@ -94,16 +95,15 @@ class SignalInputType(str, Enum):
     return self.value
 
 
-SIGNAL_COMPUTE_TYPE_TO_VALID_DTYPES: dict[SignalInputType, list[DataType]] = {
+SIGNAL_TYPE_TO_VALID_DTYPES: dict[SignalInputType, list[DataType]] = {
   SignalInputType.TEXT: [DataType.STRING, DataType.STRING_SPAN],
-  SignalInputType.TEXT_EMBEDDING: [DataType.EMBEDDING],
   SignalInputType.IMAGE: [DataType.BINARY],
 }
 
 
-def signal_compute_type_supports_dtype(input_type: SignalInputType, dtype: DataType) -> bool:
+def signal_type_supports_dtype(input_type: SignalInputType, dtype: DataType) -> bool:
   """Returns True if the signal compute type supports the dtype."""
-  return dtype in SIGNAL_COMPUTE_TYPE_TO_VALID_DTYPES[input_type]
+  return dtype in SIGNAL_TYPE_TO_VALID_DTYPES[input_type]
 
 
 Bin = tuple[str, Optional[Union[float, int]], Optional[Union[float, int]]]

lilac/server.py

@@ -117,7 +117,7 @@ def startup() -> None:
 
   if repo_id:
     # Copy datasets.
-    spaces_data_dir = os.path.join('data')
+    spaces_data_dir = 'data'
     datasets = list_datasets(spaces_data_dir)
     for dataset in datasets:
       spaces_dataset_output_dir = get_dataset_output_dir(spaces_data_dir, dataset.namespace,

lilac/signals/concept_scorer.py

@@ -7,14 +7,17 @@ from typing_extensions import override
 from ..auth import UserInfo
 from ..concepts.concept import DEFAULT_NUM_NEG_EXAMPLES, DRAFT_MAIN, ConceptColumnInfo, ConceptModel
 from ..concepts.db_concept import DISK_CONCEPT_MODEL_DB, ConceptModelDB
-from ..embeddings.vector_store import VectorStore
-from ..schema import Field, Item, RichData, VectorKey, field
-from .signal import TextEmbeddingModelSignal
+from ..data.dataset_utils import lilac_span
+from ..embeddings.vector_store import VectorDBIndex
+from ..schema import Field, Item, PathKey, RichData, SignalInputType, field
+from ..signals.signal import VectorSignal
 
 
-class ConceptScoreSignal(TextEmbeddingModelSignal):
+class ConceptScoreSignal(VectorSignal):
   """Compute scores along a given concept for documents."""
   name = 'concept_score'
+  input_type = SignalInputType.TEXT
+
   display_name = 'Concept'
 
   namespace: str
@@ -33,10 +36,16 @@ class ConceptScoreSignal(TextEmbeddingModelSignal):
 
   @override
   def fields(self) -> Field:
-    return field(
-      'float32',
-      bins=[('Not in concept', None, 0.5), ('In concept', 0.5, None)],
-    )
+    return field(fields=[
+      field(
+        dtype='string_span',
+        fields={
+          'score': field(
+            'float32',
+            bins=[('Not in concept', None, 0.5), ('In concept', 0.5, None)],
+          )
+        })
+    ])
 
   def set_column_info(self, column_info: ConceptColumnInfo) -> None:
     """Set the dataset info for this signal."""
@@ -63,26 +72,34 @@ class ConceptScoreSignal(TextEmbeddingModelSignal):
     return concept_model.score(self.draft, data)
 
   @override
-  def vector_compute(self, keys: Iterable[VectorKey],
-                     vector_store: VectorStore) -> Iterable[Optional[Item]]:
+  def vector_compute(self, keys: Iterable[PathKey],
+                     vector_index: VectorDBIndex) -> Iterable[Optional[Item]]:
     concept_model = self._get_concept_model()
-    embeddings = vector_store.get(keys)
-    return concept_model.score_embeddings(self.draft, embeddings).tolist()
+    all_vector_spans = vector_index.get(keys)
+    # TODO(smilkov): Do this with batched computation.
+    for vector_spans in all_vector_spans:
+      embeddings = np.array([vector_span['vector'] for vector_span in vector_spans])
+      scores = concept_model.score_embeddings(self.draft, embeddings)
+      res: Item = []
+      for vector_span, score in zip(vector_spans, scores):
+        start, end = vector_span['span']
+        res.append(lilac_span(start, end, {'score': score}))
+      yield res
 
   @override
   def vector_compute_topk(
       self,
       topk: int,
-      vector_store: VectorStore,
-      keys: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, Optional[Item]]]:
+      vector_index: VectorDBIndex,
+      keys: Optional[Iterable[PathKey]] = None) -> list[tuple[PathKey, Optional[Item]]]:
     concept_model = self._get_concept_model()
     query: np.ndarray = concept_model.coef(self.draft)
-    topk_keys = [key for key, _ in vector_store.topk(query, topk, keys)]
-    return list(zip(topk_keys, self.vector_compute(topk_keys, vector_store)))
+    topk_keys = [key for key, _ in vector_index.topk(query, topk, keys)]
+    return list(zip(topk_keys, self.vector_compute(topk_keys, vector_index)))
 
   @override
   def key(self, is_computed_signal: Optional[bool] = False) -> str:
     # NOTE: The embedding is a value so already exists in the path structure. This means we do not
     # need to provide the name as part of the key, which still guarantees uniqueness.
     version = f'/v{self._get_concept_model().version}' if is_computed_signal else ''
-    return f'{self.namespace}/{self.concept_name}{version}'
+    return f'{self.namespace}/{self.concept_name}/{self.embedding}{version}'

lilac/signals/lang_detection.py

@@ -1,6 +1,6 @@
 """Language detection of a document."""
 import re
-from typing import TYPE_CHECKING, Iterable, Optional, cast
+from typing import Any, Iterable, Optional, cast
 
 from pydantic import Field as PydanticField
 from typing_extensions import override
@@ -10,9 +10,7 @@ from ..schema import Field, Item, RichData, SignalInputType, field
 from .signal import TextSignal
 
 LANG_CODE = 'lang_code'
-
-if TYPE_CHECKING:
-  import langdetect
+TEXT_LEN_THRESHOLD = 25
 
 
 class LangDetectionSignal(TextSignal):
@@ -27,12 +25,18 @@ class LangDetectionSignal(TextSignal):
   display_name = 'Language detection'
 
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   split_by_paragraph: bool = PydanticField(
     default=False, description='Compute language scores for each paragraph.')
 
-  _model: Optional['langdetect.detect'] = None
+  def _detect(self, text: str, langdetect: Any) -> Optional[str]:
+
+    if len(text) < TEXT_LEN_THRESHOLD:
+      return 'TOO_SHORT'
+    try:
+      return langdetect.detect(text)
+    except langdetect.LangDetectException:
+      return None
 
   @override
   def setup(self) -> None:
@@ -42,7 +46,6 @@ class LangDetectionSignal(TextSignal):
     except ImportError:
       raise ImportError('Could not import the "langdetect" python package. '
                         'Please install it with `pip install langdetect`.')
-    self._model = langdetect.detect
 
   @override
   def fields(self) -> Field:
@@ -52,9 +55,6 @@ class LangDetectionSignal(TextSignal):
 
   @override
   def compute(self, data: Iterable[RichData]) -> Iterable[Optional[Item]]:
-    if not self._model:
-      raise RuntimeError('Language detection model is not initialized.')
-
     import langdetect
     data = cast(Iterable[str], data)
     # Split on paragraphs.
@@ -62,10 +62,7 @@ class LangDetectionSignal(TextSignal):
 
     for text in data:
       if not self.split_by_paragraph:
-        try:
-          yield self._model(text)
-        except langdetect.LangDetectException:
-          yield None
+        yield self._detect(text, langdetect)
         continue
 
       prev_end = 0
@@ -75,20 +72,16 @@ class LangDetectionSignal(TextSignal):
         text_span = text[prev_end:start]
         text_span = text_span.strip()
         if text_span:
-          try:
-            lang_code = self._model(text_span)
+          lang_code = self._detect(text_span, langdetect)
+          if lang_code:
             result.append(lilac_span(prev_end, start, {LANG_CODE: lang_code}))
-          except langdetect.LangDetectException:
-            pass
         prev_end = end
 
       # Process the last chunk.
       text_span = text[prev_end:]
       if text_span.strip():
-        try:
-          lang_code = self._model(text_span)
+        lang_code = self._detect(text_span, langdetect)
+        if lang_code:
           result.append(lilac_span(prev_end, len(text), {LANG_CODE: lang_code}))
-        except langdetect.LangDetectException:
-          pass
 
       yield result

lilac/signals/minhash_dup.py

@@ -17,7 +17,7 @@ from scipy.integrate import quad as integrate
 from tqdm import tqdm
 
 SEED = 42
-NON_ALPHA = re.compile('[^A-Za-z_0-9]')
+WHITESPACE = re.compile(r'\s+')
 RNG = np.random.RandomState(SEED)
 MAX_HASH = np.uint64((1 << 32) - 1)
 MERSENNE_PRIME = np.uint64((1 << 61) - 1)
@@ -72,7 +72,7 @@ def _embed_func(
     The hash values in each range and the index.
   """
   hashvalues = np.ones(num_perm, dtype=np.uint64) * MAX_HASH
-  tokens = {' '.join(t) for t in _ngrams(NON_ALPHA.split(content), ngram_size, min_ngram_size)}
+  tokens = {' '.join(t) for t in _ngrams(WHITESPACE.split(content), ngram_size, min_ngram_size)}
   hv = np.array([_sha1_hash32(token.encode('utf-8')) for token in tokens],
                 dtype=np.uint64)  # noqa: E501
   a, b = permutations

lilac/signals/near_dup.py

@@ -24,7 +24,6 @@ class NearDuplicateSignal(TextSignal):
   display_name = 'Near duplicate documents'
 
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   threshold: float = PydanticField(
     default=0.75,

lilac/signals/ner.py

@@ -23,7 +23,6 @@ class SpacyNER(TextSignal):
   model: str = PydanticField(title='SpaCy package name or model path.', default='en_core_web_sm')
 
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   _nlp: Optional['spacy.language.Language'] = None
 

lilac/signals/pii.py

@@ -25,7 +25,6 @@ class PIISignal(TextSignal):
   display_name = 'Personal Information (PII)'
 
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   @override
   def fields(self) -> Field:

lilac/signals/semantic_similarity.py

@@ -5,13 +5,14 @@ import numpy as np
 from scipy.interpolate import interp1d
 from typing_extensions import override
 
+from ..data.dataset_utils import lilac_span
 from ..embeddings.embedding import EmbedFn, get_embed_fn
-from ..embeddings.vector_store import VectorStore
-from ..schema import Field, Item, RichData, VectorKey, field
-from .signal import TextEmbeddingModelSignal
+from ..embeddings.vector_store import VectorDBIndex
+from ..schema import Field, Item, PathKey, RichData, SignalInputType, field
+from .signal import VectorSignal
 
 
-class SemanticSimilaritySignal(TextEmbeddingModelSignal):
+class SemanticSimilaritySignal(VectorSignal):
   """Compute semantic similarity for a query and a document.
 
   \
@@ -20,6 +21,7 @@ class SemanticSimilaritySignal(TextEmbeddingModelSignal):
   """
   name = 'semantic_similarity'
   display_name = 'Semantic Similarity'
+  input_type = SignalInputType.TEXT
 
   query: str
 
@@ -32,15 +34,13 @@ class SemanticSimilaritySignal(TextEmbeddingModelSignal):
   def __init__(self, query: Union[str, bytes], embedding: str, **kwargs: Any):
     if isinstance(query, bytes):
       raise ValueError('Image queries are not yet supported for SemanticSimilarity.')
-
-    super().__init__(query=query, embedding=embedding, **kwargs)
-
+    super().__init__(query=query, embedding=embedding, **kwargs)  # type: ignore
     # TODO(nsthorat): The embedding cls might have arguments. This needs to be resolved.
     self._embed_fn = get_embed_fn(embedding)
 
   @override
   def fields(self) -> Field:
-    return field('float32')
+    return field(fields=[field(dtype='string_span', fields={'score': 'float32'})])
 
   def _get_search_embedding(self) -> np.ndarray:
     """Return the embedding for the search text."""
@@ -51,26 +51,32 @@ class SemanticSimilaritySignal(TextEmbeddingModelSignal):
 
   @override
   def compute(self, data: Iterable[RichData]) -> Iterable[Optional[Item]]:
-    text_embeddings = self._embed_fn(data)
-    similarities = text_embeddings.dot(self._get_search_embedding()).reshape(-1)
-    return similarities.tolist()
+    embeddings = self._embed_fn(data)
+    scores = embeddings.dot(self._get_search_embedding()).reshape(-1)
+    for text, score in zip(data, scores):
+      yield [lilac_span(0, len(text), {'score': score})]
 
   @override
-  def vector_compute(self, keys: Iterable[VectorKey],
-                     vector_store: VectorStore) -> Iterable[Optional[Item]]:
-    text_embeddings = vector_store.get(keys)
-    similarities = text_embeddings.dot(self._get_search_embedding()).reshape(-1)
-    # Clip the similarities since float precision can cause these to be barely outside the range and
-    # throw an exception with interp1d.
-    similarities = np.clip(similarities, -1., 1.)
-    return self._interpolate_fn(similarities).tolist()
+  def vector_compute(self, keys: Iterable[PathKey],
+                     vector_index: VectorDBIndex) -> Iterable[Optional[Item]]:
+    all_vector_spans = vector_index.get(keys)
+    query = self._get_search_embedding()
+    # TODO(smilkov): Do this with batched computation.
+    for vector_spans in all_vector_spans:
+      embeddings = np.array([vector_span['vector'] for vector_span in vector_spans])
+      scores = embeddings.dot(query).reshape(-1)
+      res: Item = []
+      for vector_span, score in zip(vector_spans, scores):
+        start, end = vector_span['span']
+        res.append(lilac_span(start, end, {'score': score}))
+      yield res
 
   @override
   def vector_compute_topk(
       self,
       topk: int,
-      vector_store: VectorStore,
-      keys: Optional[Iterable[VectorKey]] = None) -> list[tuple[VectorKey, Optional[Item]]]:
+      vector_index: VectorDBIndex,
+      keys: Optional[Iterable[PathKey]] = None) -> list[tuple[PathKey, Optional[Item]]]:
     query = self._get_search_embedding()
-    topk_keys = [key for key, _ in vector_store.topk(query, topk, keys)]
-    return list(zip(topk_keys, self.vector_compute(topk_keys, vector_store)))
+    topk_keys = [key for key, _ in vector_index.topk(query, topk, keys)]
+    return list(zip(topk_keys, self.vector_compute(topk_keys, vector_index)))

lilac/signals/signal.py

@@ -6,8 +6,8 @@ from typing import Any, ClassVar, Iterable, Optional, Sequence, Type, TypeVar, U
 from pydantic import BaseModel, Extra, validator
 from typing_extensions import override
 
-from ..embeddings.vector_store import VectorStore
-from ..schema import Field, Item, RichData, SignalInputType, VectorKey, field
+from ..embeddings.vector_store import VectorDBIndex
+from ..schema import Field, Item, PathKey, RichData, SignalInputType, field
 
 EMBEDDING_KEY = 'embedding'
 
@@ -19,12 +19,8 @@ class Signal(abc.ABC, BaseModel):
   # The display name is just used for rendering in the UI.
   display_name: ClassVar[Optional[str]]
 
-  # The input type is used to populate the UI for signals that require other signals. For example,
-  # if a signal is an TextEmbeddingModelSignal, it computes over embeddings, but it's input type
-  # will be text.
+  # The input type is used to populate the UI to determine what the signal accepts as input.
   input_type: ClassVar[SignalInputType]
-  # The compute type defines what should be passed to compute().
-  compute_type: ClassVar[SignalInputType]
 
   # The signal_name will get populated in init automatically from the class name so it gets
   # serialized and the signal author doesn't have to define both the static property and the field.
@@ -79,40 +75,6 @@ class Signal(abc.ABC, BaseModel):
     """
     raise NotImplementedError
 
-  def vector_compute(self, keys: Iterable[VectorKey],
-                     vector_store: VectorStore) -> Iterable[Optional[Item]]:
-    """Compute the signal for an iterable of keys that point to documents or images.
-
-    Args:
-      keys: An iterable of value ids (at row-level or lower) to lookup precomputed embeddings.
-      vector_store: The vector store to lookup pre-computed embeddings.
-
-    Returns
-      An iterable of items. Sparse signals should return "None" for skipped inputs.
-    """
-    raise NotImplementedError
-
-  def vector_compute_topk(
-      self,
-      topk: int,
-      vector_store: VectorStore,
-      keys: Optional[Iterable[VectorKey]] = None) -> Sequence[tuple[VectorKey, Optional[Item]]]:
-    """Return signal results only for the top k documents or images.
-
-    Signals decide how to rank each document/image in the dataset, usually by a similarity score
-    obtained via the vector store.
-
-    Args:
-      topk: The number of items to return, ranked by the signal.
-      vector_store: The vector store to lookup pre-computed embeddings.
-      keys: Optional iterable of row ids to restrict the search to.
-
-    Returns
-      A list of (key, signal_output) tuples containing the `topk` items. Sparse signals should
-      return "None" for skipped inputs.
-    """
-    raise NotImplementedError
-
   def key(self, is_computed_signal: Optional[bool] = False) -> str:
     """Get the key for a signal.
 
@@ -155,7 +117,6 @@ def _args_key_from_dict(args_dict: dict[str, Any]) -> str:
 class TextSplitterSignal(Signal):
   """An interface for signals that compute over text."""
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   @override
   def fields(self) -> Field:
@@ -166,7 +127,6 @@ class TextSplitterSignal(Signal):
 class TextSignal(Signal):
   """An interface for signals that compute over text."""
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   @override
   def key(self, is_computed_signal: Optional[bool] = False) -> str:
@@ -179,7 +139,6 @@ class TextSignal(Signal):
 class TextEmbeddingSignal(TextSignal):
   """An interface for signals that compute embeddings for text."""
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   _split = True
 
@@ -196,38 +155,43 @@ class TextEmbeddingSignal(TextSignal):
     return field(fields=[field('string_span', fields={EMBEDDING_KEY: 'embedding'})])
 
 
-class TextEmbeddingModelSignal(TextSignal):
-  """An interface for signals that take embeddings and produce items."""
-  input_type = SignalInputType.TEXT
-  # compute() takes embeddings, while it operates over text fields by transitively computing splits
-  # and embeddings.
-  compute_type = SignalInputType.TEXT_EMBEDDING
-
+class VectorSignal(Signal, abc.ABC):
+  """An interface for signals that can compute items given vector inputs."""
   embedding: str
-  _embedding_signal: Optional[TextEmbeddingSignal] = None
 
-  def __init__(self, **kwargs: Any):
-    super().__init__(**kwargs)
+  def vector_compute(self, keys: Iterable[PathKey],
+                     vector_index: VectorDBIndex) -> Iterable[Optional[Item]]:
+    """Compute the signal for an iterable of keys that point to documents or images.
 
-    # Validate the embedding signal is registered and the correct type.
-    # TODO(nsthorat): Allow arguments passed to the embedding signal.
-    self._embedding_signal = get_signal_by_type(self.embedding, TextEmbeddingSignal)()
+    Args:
+      keys: An iterable of value ids (at row-level or lower) to lookup precomputed embeddings.
+      vector_index: The vector index to lookup pre-computed embeddings.
 
-  def get_embedding_signal(self) -> TextEmbeddingSignal:
-    """Return the embedding signal."""
-    assert self._embedding_signal is not None
-    return self._embedding_signal
+    Returns
+      An iterable of items. Sparse signals should return "None" for skipped inputs.
+    """
+    raise NotImplementedError
 
-  @override
-  def key(self, is_computed_signal: Optional[bool] = False) -> str:
-    # NOTE: The embedding and split already exists in the path structure. This means we do not
-    # need to provide the signal names as part of the key, which still guarantees uniqueness.
+  def vector_compute_topk(
+      self,
+      topk: int,
+      vector_index: VectorDBIndex,
+      keys: Optional[Iterable[PathKey]] = None) -> Sequence[tuple[PathKey, Optional[Item]]]:
+    """Return signal results only for the top k documents or images.
 
-    args_dict = self.dict(exclude_unset=True)
-    if 'signal_name' in args_dict:
-      del args_dict['signal_name']
-    del args_dict['embedding']
-    return self.name + _args_key_from_dict(args_dict)
+    Signals decide how to rank each document/image in the dataset, usually by a similarity score
+    obtained via the vector store.
+
+    Args:
+      topk: The number of items to return, ranked by the signal.
+      vector_index: The vector index to lookup pre-computed embeddings.
+      keys: Optional iterable of row ids to restrict the search to.
+
+    Returns
+      A list of (key, signal_output) tuples containing the `topk` items. Sparse signals should
+      return "None" for skipped inputs.
+    """
+    raise NotImplementedError
 
 
 Tsignal = TypeVar('Tsignal', bound=Signal)

lilac/signals/splitters/chunk_splitter.py

@@ -41,7 +41,7 @@ from ..signal import TextSplitterSignal
 
 TextChunk = tuple[str, tuple[int, int]]
 
-DEFAULT_SEPARATORS = ['\n\n', '\n', ' ', '']
+DEFAULT_SEPARATORS = ['```', '\n\n', '\n', ' ', '']
 CHUNK_SIZE = 400
 CHUNK_OVERLAP = 50
 
@@ -99,10 +99,24 @@ def _sep_split(text: str, separator: str) -> list[TextChunk]:
 
   offset = 0
   chunks: list[TextChunk] = []
+  open_code_block = False
   end_index = text.find(separator, offset)
 
   while end_index >= 0:
-    chunks.append((text[offset:end_index], (offset, end_index)))
+    if separator == '```':
+      # We want to keep the code block seperators as part of the text chunk.
+      start = max(0, offset - len(separator))
+      if open_code_block:
+        end = end_index + len(separator)
+        open_code_block = False
+      else:
+        end = end_index
+        open_code_block = True
+    else:
+      start = offset
+      end = end_index
+
+    chunks.append((text[start:end], (start, end)))
     offset = end_index + len(separator)
     end_index = text.find(separator, offset)
 
@@ -120,9 +134,8 @@ def split_text(text: str,
   """Split incoming text and return chunks."""
 
   def _merge_splits(splits: Iterable[TextChunk], separator: str) -> list[TextChunk]:
-    # We now want to combine these smaller pieces into medium size
-    # chunks to send to the LLM.
-    separator_len = length_function(separator)
+    # We now want to combine these smaller pieces into medium size chunks to send to the LLM.
+    separator_len = 0 if separator == '```' else length_function(separator)
 
     docs: list[TextChunk] = []
     current_doc: list[TextChunk] = []
@@ -157,12 +170,14 @@ def split_text(text: str,
   final_chunks: list[TextChunk] = []
   # Get appropriate separator to use
   separator = separators[-1]
-  for _s in separators:
+  new_separators: list[str] = []
+  for i, _s in enumerate(separators):
     if _s == '':
       separator = _s
       break
     if _s in text:
       separator = _s
+      new_separators = separators[i + 1:]
       break
   # Now that we have the separator, split the text.
   splits = _sep_split(text, separator)
@@ -177,7 +192,8 @@ def split_text(text: str,
         merged_text = _merge_splits(good_splits, separator)
         final_chunks.extend(merged_text)
         good_splits = []
-      other_chunks = split_text(text_chunk, chunk_size, chunk_overlap, separators, length_function)
+      other_chunks = split_text(text_chunk, chunk_size, chunk_overlap, new_separators,
+                                length_function)
       # Adjust the offsets of the other chunks.
       other_chunks = [(t, (s + start, e + start)) for t, (s, e) in other_chunks]
       final_chunks.extend(other_chunks)
@@ -188,6 +204,9 @@ def split_text(text: str,
 
 
 def _join_chunks(chunks: list[TextChunk], separator: str) -> Optional[TextChunk]:
+  if separator == '```':
+    # Code blocks already have the separator.
+    separator = ''
   text = separator.join([text for text, _ in chunks])
   text = text.strip()
   if text == '':

lilac/signals/substring_search.py

@@ -14,7 +14,6 @@ class SubstringSignal(Signal):
   name = 'substring_search'
   display_name = 'Substring Search'
   input_type = SignalInputType.TEXT
-  compute_type = SignalInputType.TEXT
 
   query: str
 

lilac/signals/text_statistics.py

@@ -13,6 +13,7 @@ SPACY_BATCH_SIZE = 128
 NUM_CHARS = 'num_characters'
 READABILITY = 'readability'
 TYPE_TOKEN_RATIO = 'log(type_token_ratio)'
+FRAC_NON_ASCII = 'frac_non_ascii'
 
 if TYPE_CHECKING:
   from spacy import Language
@@ -28,11 +29,14 @@ class TextStatisticsSignal(TextSignal):
 
   @override
   def fields(self) -> Field:
-    return field(fields={
-      NUM_CHARS: 'int32',
-      READABILITY: 'float32',
-      TYPE_TOKEN_RATIO: 'float32',
-    })
+    return field(
+      fields={
+        NUM_CHARS: 'int32',
+        READABILITY: 'float32',
+        TYPE_TOKEN_RATIO: 'float32',
+        FRAC_NON_ASCII: field(
+          'float32', bins=[('Low', None, 0.15), ('Medium', 0.15, 0.3), ('High', 0.3, None)])
+      })
 
   @override
   def setup(self) -> None:
@@ -71,15 +75,27 @@ class TextStatisticsSignal(TextSignal):
       # available statistics.
       corpus = textacy.corpus.Corpus(lang=self._lang, data=batch)
       for doc in cast(Iterable['Doc'], corpus):
-        if not len(doc):
+        if not doc or not doc.text.strip():
           yield None
           continue
-        readability = text_stats.readability.automated_readability_index(doc)
-        ttr = text_stats.diversity.log_ttr(doc)
-        num_chars = text_stats.basics.n_chars(doc)
+        try:
+          readability = text_stats.readability.automated_readability_index(doc)
+        except ZeroDivisionError:
+          readability = None
+        try:
+          ttr = text_stats.diversity.log_ttr(doc)
+        except ValueError:
+          ttr = None
+        num_chars = len(doc.text)
+        num_non_ascii = 0
+        for c in doc.text:
+          if ord(c) >= 128:
+            num_non_ascii += 1
+        frac_non_ascii = num_non_ascii / num_chars if num_chars else 0
 
         yield {
           NUM_CHARS: num_chars,
           READABILITY: readability,
           TYPE_TOKEN_RATIO: ttr,
+          FRAC_NON_ASCII: frac_non_ascii
         }

lilac/web/_app/immutable/assets/ConceptView.98f1ad48.css

@@ -0,0 +1 @@
+button.svelte-d3v0kx{width:100%;padding:.5rem 1rem;text-align:left;--tw-text-opacity:1;color:rgb(31 41 55 / var(--tw-text-opacity))}button.svelte-d3v0kx:hover{--tw-bg-opacity:1;background-color:rgb(229 231 235 / var(--tw-bg-opacity));--tw-text-opacity:1;color:rgb(0 0 0 / var(--tw-text-opacity))}button[data-active=true].svelte-d3v0kx{--tw-bg-opacity:1;background-color:rgb(209 213 219 / var(--tw-bg-opacity));--tw-text-opacity:1;color:rgb(0 0 0 / var(--tw-text-opacity))}.bx--tag{margin:0}.concept-score-pill .bx--tooltip__label{margin-right:.25rem;display:inline-block;height:100%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:5rem}.concept-score-pill .bx--tooltip__content{display:flex;flex-direction:column;align-items:center}table.svelte-zc5pc5 td.svelte-zc5pc5{padding:.25rem .5rem}.embedding-badge-nolabel .bx--tooltip__label .bx--tooltip__trigger{margin:0}.embedding-badge-nolabel .bx--tag__custom-icon{margin-right:0}.more-button .bx--btn{height:1.5rem;width:12rem}.named-value-name.svelte-1689hje{max-width:15rem}.highlight-span.svelte-8ox5pu{padding-top:1.5px;padding-bottom:1.5px}.highlight-span pre{--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity));font-size:.875rem;line-height:1.25rem}.highlight-span p,.highlight-span pre{margin-top:.75rem;margin-bottom:.75rem}.highlight-span p:first-child{display:inline!important}.highlight-span p:last-child{display:inline!important}.highlight-span p,.highlight-span h1{background-color:inherit}.highlight-span p{font-size:.875rem;line-height:1.25rem;font-weight:inherit}.dataset-link.bx--btn{min-height:0px}