Initial commit.

.gitignore

@@ -0,0 +1 @@
+*/__pycache__/

Makefile

@@ -0,0 +1,16 @@
+.PHONY: install style test
+
+PYTHON := python
+CHECK_DIRS := llmlingua tests
+
+install:
+	@${PYTHON} setup.py bdist_wheel
+	@${PYTHON} -m pip install dist/sdtools*
+
+style:
+	black $(CHECK_DIRS)
+	isort -rc $(CHECK_DIRS)
+	flake8 $(CHECK_DIRS)
+
+test:
+	@${PYTHON} -m pytest -n auto --dist=loadfile -s -v ./tests/

app.py

@@ -0,0 +1,120 @@
+
+# build the environment
+import sys
+import subprocess
+subprocess.run([sys.executable, "-m", "pip", "install", "-e", "."])
+
+# import the required libraries
+import gradio as gr
+import json
+from llmlingua import PromptCompressor
+import tiktoken
+
+# load the pre-trained models
+compressors = {
+    "xlm-roberta-large": PromptCompressor(
+        model_name="microsoft/llmlingua-2-xlm-roberta-large-meetingbank",
+        use_llmlingua2=True
+    ),
+    "mbert-base": PromptCompressor(
+        model_name="microsoft/llmlingua-2-bert-base-multilingual-cased-meetingbank",
+        use_llmlingua2=True
+    )
+}
+tokenizer = tiktoken.encoding_for_model("gpt-4")
+
+with open('data/examples_MeetingBank.json', 'r') as f:
+    examples = json.load(f) # list of examples, each example is a list of 3 group of values: idx (), original prompt (str), QA pairs (list of list of 2 strings)
+original_prompt_list = [[s["original_prompt"]] for s in examples]
+qa_list = [s["QA_pairs"] for s in examples]
+
+def compress(original_prompt, compression_rate, base_model="xlm-roberta-large", force_tokens=['\n'], chunk_end_tokens=['.', '\n']):
+    if '\\n' in force_tokens:
+        idx = force_tokens.index('\\n')
+        force_tokens[idx] = '\n'
+
+    compressor = compressors.get(base_model, compressors["xlm-roberta-large"])
+    results = compressor.compress_prompt_llmlingua2(
+            original_prompt,
+            rate=compression_rate,
+            force_tokens=force_tokens,
+            chunk_end_tokens=chunk_end_tokens,
+            return_word_label=True,
+            drop_consecutive=True
+            )
+
+    compressed_prompt = results["compressed_prompt"]
+    n_word_compressed = len(tokenizer.encode(compressed_prompt))
+    
+    word_sep = "\t\t|\t\t"
+    label_sep = " "
+    lines = results["fn_labeled_original_prompt"].split(word_sep)
+    preserved_tokens = []
+    for line in lines:
+        word, label = line.split(label_sep)
+        preserved_tokens.append((word, '+') if label == '1' else (word, None))
+
+    return compressed_prompt, preserved_tokens, n_word_compressed
+
+
+title = "LLMLingua-2"
+header = ("""
+    <div align='center'>
+    <h1></h1>
+    <h1>LLMLingua-2: Efficient and Faithful Task-Agnostic Prompt Compression via Data Distillation</h1>
+    <h3>Zhuoshi Pan, Qianhui Wu, Huiqiang Jiang, Menglin Xia, Xufang Luo, Jue Zhang, Qingwei Lin, Victor Ruehle, Yuqing Yang, Chin-Yew Lin, H. Vicky Zhao, Lili Qiu, and Dongmei Zhang</h3>
+    <h3><a href='' target='_blank' rel='noopener'>[project page]</a><a href='' target='_blank' rel='noopener'>[paper]</a><a href='' target='_blank' rel='noopener'>[code]</a>
+    </div>
+    """
+)
+theme = "soft"
+css = """#anno-img .mask {opacity: 0.5; transition: all 0.2s ease-in-out;}
+            #anno-img .mask.active {opacity: 0.7}"""
+
+original_prompt_text = """John: So, um, I've been thinking about the project, you know, and I believe we need to, uh, make some changes. I mean, we want the project to succeed, right? So, like, I think we should consider maybe revising the timeline.
+
+Sarah: I totally agree, John. I mean, we have to be realistic, you know. The timeline is, like, too tight. You know what I mean? We should definitely extend it.
+"""
+# with gr.Blocks(title=title, theme=gr.themes.Soft(), css=css) as app:
+with gr.Blocks(title=title, css=css) as app: # 'YenLai/Superhuman' 'HaleyCH/HaleyCH_Theme' 'gradio/monochrome' ''zkunn/Alipay_Gradio_theme''
+    gr.Markdown(header)
+    with gr.Row():
+        with gr.Column(scale=3):
+            original_prompt = gr.Textbox(value=original_prompt_text, label="Original Prompt", lines=10, max_lines=10, interactive=True)
+            compressed_prompt = gr.Textbox(value='', label="Compressed Prompt", lines=10, max_lines=10, interactive=False)
+            
+        with gr.Column(scale=1):
+            base_model = gr.Radio(["xlm-roberta-large", "mbert-base"], label="Base Model", value="xlm-roberta-large", interactive=True)
+            force_tokens = gr.Dropdown(['\\n', '.', '!', '?', ','],
+                                       label="Tokens to Preserve",
+                                       value=['\\n', '.', '!', '?', ','],
+                                       multiselect=True,
+                                       interactive=True)
+            compression_rate = gr.Slider(minimum=0.1, maximum=1.0, step=0.1, value=0.7, label="Compression rate", info="after compr. / befor compr.", interactive=True)
+            n_word_original = gr.Textbox(lines=1, label="Original (GPT-4 Tokens)", interactive=False, value=len(tokenizer.encode(original_prompt_text)))
+            n_word_compressed = gr.Textbox(lines=1, label="Compressed (GPT-4 Tokens)", interactive=False)
+    button = gr.Button("⚡Click to Compress")
+    with gr.Accordion(label="Compression Details", open=False):
+        diff_text = gr.HighlightedText(label="Diff", combine_adjacent=False, show_legend=True, color_map={"+": "green"})
+
+    original_prompt.change(lambda x: len(tokenizer.encode(x)), inputs=[original_prompt], outputs=[n_word_original])
+    original_prompt.change(lambda x: ("", "", []), inputs=[original_prompt], outputs=[compressed_prompt, n_word_compressed, diff_text])
+
+    button.click(fn=compress,
+                 inputs=[original_prompt, compression_rate, base_model, force_tokens],
+                 outputs=[compressed_prompt, diff_text, n_word_compressed])
+    
+    qa_pairs = gr.DataFrame(label="GPT-4 generated QA pairs related to the original prompt:", headers=["Question", "Answer"], interactive=True,
+                            value=[["Summarize the conversation.","John suggests making changes to the project, specifically revising the timeline to ensure its success. Sarah agrees with John, acknowledging that the current timeline is too tight and supports the idea of extending it."]])
+    
+    gr.Markdown("## Examples (click to select)")
+    dataset = gr.Dataset(label="MeetingBank",
+                         components=[gr.Textbox(visible=False, max_lines=3)],
+                         samples=original_prompt_list,
+                         type="index")
+    
+    dataset.select(fn=lambda idx: (examples[idx]["original_prompt"], examples[idx]["QA_pairs"]),
+                   inputs=[dataset],
+                   outputs=[original_prompt, qa_pairs])
+        
+app.queue(max_size=10, api_open=False).launch(show_api=False)

data/examples_MeetingBank.json

@@ -0,0 +1,462 @@
+[
+    {
+        "idx": 0,
+        "original_prompt": "The report of the Civil Rights, Utilities, Economic Development and Arts Committee Agenda Item three Resolution 31669 Encouraging as a best practice the use of an individualized tenant assessment using the Fair Housing Act's discriminatory effect standards to avoid Fair Housing Act violations when criminal history is used as a screening criterion in the Landlord Screening Process, Committee recommends that the resolution be adopted as amended grade. I move to amend Resolution 31669 by substituting D four for version D three, which includes a new attachment. A And I understand Councilmember Bagshaw also has an amendment, but let's first, if we could, let me just go through the changes to the resolution since the last committee meeting. The changes are found in two recitals, as well as sections one and five are the primary changes. We added a recital that again lifts up the HUD guidance to show that a criminal history screening policy is next must serve a substantial, legitimate and nondiscriminatory interest. Another recital we included was referencing the the Seattle Fair Chance Employment Ordinance and the way they approach some of these same issues, looking at making sure that individualized assessments and prohibiting questions on initial job applications regarding an applicant's criminal record. And then in Section one, and these are changes that we worked on with stakeholders together with councilmembers, the wants office desiring to really focus on not the the impacts of this particular resolution, but for what we hope to see in the future ordinance that is going to be coming to us to to regulate this area of of housing screening practices. And it identifies the principles that came out of the Hallow recommendations. And in Section five, again, this is just clarifying that the expectation in the HUD guidance is to distinguish between criminal conduct, that that indicates a demonstrable risk to residents safety and conduct that does not the resolution itself, whereas it's really focused on encouraging landlords to to follow HUD guidance that has been recently released regarding criminal records. The separate sections do. A couple of different things. Sections one and two, again, focus specifically on the future legislation that we expect to be coming out of the mayors task force. The. The next section basically says that we endorse practices that landlords should not automatically exclude individuals for housing on the basis of prior event arrests. The further sections refer refer to the process that the Office of Housing has facilitated to create procedures to select tenant screening agency guidelines for property management and affordable housing. Another section recommends that the that a landlord should not rely on records that cannot be reported by consumer reporting agencies under state law. And another section focuses on the Office of Civil Rights efforts to basically do enforcement of existing fair housing laws through testing, investigation of charges and other other means. The final section requests that as OCR when determining whether or not a complaint of housing discrimination based on the use of criminal history, whether or not there should they should it ask them to seek to determine whether or not there's a disparate impact? So that's an overview of both the resolution and the the changes that have been made since the committee discussion and vote on June 3rd. And I don't I may have started talking before I had a second. May I add a second? All right, great. Those in favor of supporting the substitute version D for 4d3ii in a OC. So we have the substitute amendment before us. Councilmember Bagshaw will move to further amend the resolution, but before consideration of the amendment, we have to move to suspend the rules because we've got we received the text for the amendment after the , I believe, noon deadline today. So I moved to suspend Council Rule three A6 relating to presentation of full council amendments before 12 noon, checking all those in favor of the motion carries and we can proceed with consideration of the proposed amendment. Great. Thank you very much. What I am proposing is the addition in our whereas section two recognize what we all worked on a year ago called the Certificate of Restoration of Opportunity or the acronym was CROP. And it really and it's designed to address what the gentleman in the front row had talked about earlier today during public testimony , the state legislature passed unanimously out of both houses of the Act around Certificate of Restoration of opportunity. And what it is designed to do is to offer potential public and private employers or housing providers concrete and objective information about an individual who has served his or her time in prison and is now been released. And what we're really wanting to do here is to reintegrate individuals who have had a previous criminal history to provide housing and employment opportunities so that whereas that I am recommending we ensure it comes right out of the bill. And it would say that in an act relating to certificates of restoration of opportunity that will offer potential public and private employers or housing providers concrete and objective information about an individual under consideration for an opportunity. These certificates can facilitate the successful societal reintegration of individuals with a criminal history whose behavior demonstrates that they are taking responsibility for their past criminal conduct, pursuing a positive, law abiding future. So I'm just suggesting we add this, that it refers to this legislation, which I will hope a court will provide certificate, a restoration of opportunity, and an individual has something else in his or her hand to help him get a job or housing. So we have a first. We moved it and we second it as well. No. Okay. We have a move in a second. Now, all those in favor of the amendment to the resolution say I, I, I opposed me. And now we will have the full version before us to vote on any comments. Comment. Sorry, I have just some closing statements. I just really I think it's so important that landlords, housing providers in this community understand what the law is when it comes to developing policies and practices for making decisions based on criminal history. We know that we're not likely to have the ordinance that will do this work and until after the Mayors for Fair Chance Housing Committee will be reconvened in July, and they will have a series of meetings before they bring to us recommendations for for an ordinance. And so in the interim, it's really important that we lift up the the policies that HUD is currently currently promulgating and making sure that both landlords are engaged with the policy direction that the that the city is going to be pursuing in the future, as well as protecting themselves from fair housing complaints today. So with that those in favor of adopting the resolution vote i. I. Those oppose vote no. The motion carries the resolution is adopted and the chair will sign it. And now we can read items for through eight together.",
+        "QA_pairs": [
+            [
+                "What is the agenda item three resolution 31669 about?",
+                "Encouraging individualized tenant assessment."
+            ],
+            [
+                "What is the amendment proposed by Councilmember Bagshaw?",
+                "Addition in the \"whereas\" section."
+            ],
+            [
+                "What are the primary changes in the resolution since the last committee meeting?",
+                "Changes in two recitals and sections one and five."
+            ],
+            [
+                "What does the HUD guidance emphasize?",
+                "Distinguishing between criminal conduct that indicates a risk to residents safety and conduct that does not."
+            ],
+            [
+                "What does one of the sections of the resolution endorse?",
+                "Landlords should not automatically exclude individuals for housing on the basis of prior event arrests."
+            ],
+            [
+                "What does the final section request from the OCR?",
+                "To determine whether there's a disparate impact in complaints of housing discrimination based on the use of criminal history."
+            ],
+            [
+                "What is the Certificate of Restoration of Opportunity (CROP)?",
+                "A certificate providing concrete and objective information about an individual who has served his or her time in prison."
+            ],
+            [
+                "What is the purpose of CROP?",
+                "To facilitate the successful societal reintegration of individuals with a criminal history."
+            ],
+            [
+                "What is the resolution's focus?",
+                "Encouraging landlords to follow HUD guidance regarding criminal records."
+            ],
+            [
+                "What is the outcome of the vote on the resolution?",
+                "The resolution is adopted."
+            ]
+        ]
+    },
+    {
+        "idx": 1,
+        "original_prompt": "Madam Court, could you please read docket 1239? Certainly. Docket 1239. The Committee on Government Operations, to which was referred on December 1st, 2021, docket number 1239 message an order authorizing the creation of a sheltered market program in conformity with the requirements of general laws. Chapter 30 B Section 18. This authorization applies to contracts for goods, professional services and support services. This authorization is for no more than six contracts, which must be awarded by June 30th, 2022. This sheltered market program shall be available for disadvantaged, minority and women only vendors, for whom there is a demonstrated substantial disparity in the city's 2020 disparities. Study submits a report recommending the order ought to pass. Thank you so much, Madam Clerk. The Chair recognizes Councilor Edwards, chair of the committee. Councilor Edwards. You have the floor. This is this is actually a matter, I believe, sponsored by the. Mayor in Cannes. In conformance with the recommendations from the disparity study and making sure that we opt in to this this pilot program under mass general laws 30 Section 18. Again, it's really just following the recommendations of an already studied issue, which which demonstrates a disparity between minority contractors or women contractors receiving contracts in the city of Boston. So this would allow for us to shepherd and move these six contracts to those already designated groups who have a disadvantage. And I think it's. Really fulfilling a promise. Of making sure that we go through and make sure all aspects of the city government, including the financial benefits, are accessible to people in the city of Boston. I recommend that this pass and I hope that my colleagues will vote for it. Thank you. Thank you so much. Councilor Edward seeks acceptance of the committee report and passage of Docket 1239. Madam Court, could you please call the roll? Certainly. Docket 1239. Councilor Arroyo. Yes. Councilor Arroyo. Yes. Councilor Baker. Councilor Baker. Councilor. Councilor Barker. Council Braden. Councilor Braden. Councilor Campbell. Councilor Campbell. Yes. Councilor Edwards. Yes. Councilor Sabby. George. Councilor Sabby. George. He has Councilor Flaherty. Councilor Flaherty as Councilor Flynn. Councilor Flynn. Yes. Councilor Jane. Yes. Councilor Janey. As Councilor me here. Councilor me here as Councilor Murphy. Councilor Murphy. Yes. And Councilor O'Malley. Yes. Councilor O'Malley. Yes. Madam President, do I get number 1239 has received unanimous vote. Thank you so much. Dockett 1239 has passed and now we will move on to matters recently heard for possible action. Madam Clerk, if you could please read docket 0863. Certainly Docket 0863 order for hearing to discuss pest control and illegal dumping in the city of Boston.",
+        "QA_pairs": [
+            [
+                "Who read docket 1239?",
+                "Madam Court."
+            ],
+            [
+                "What is the purpose of docket 1239?",
+                "Creation of a sheltered market program."
+            ],
+            [
+                "Who is the sheltered market program for?",
+                "Disadvantaged, minority and women vendors."
+            ],
+            [
+                "How many contracts does the authorization apply to?",
+                "No more than six contracts."
+            ],
+            [
+                "When must the contracts be awarded by?",
+                "June 30th, 2022."
+            ],
+            [
+                "Who sponsored the matter discussed in the meeting?",
+                "The Mayor in Cannes."
+            ],
+            [
+                "Who recommended the passage of Docket 1239?",
+                "Councilor Edwards."
+            ],
+            [
+                "What was the result of the vote on Docket 1239?",
+                "It received unanimous vote."
+            ],
+            [
+                "What was the next action after Docket 1239 passed?",
+                "Move on to matters recently heard."
+            ],
+            [
+                "What is the subject of Docket 0863?",
+                "Discuss pest control and illegal dumping."
+            ]
+        ]
+    },
+    {
+        "idx": 2,
+        "original_prompt": "Item 15, report from City Manager Recommendation to adopt three resolutions. First, to join the Victory Pace program. Second, to join the California first program. And number three, consenting to to inclusion of certain properties within the jurisdiction in the California Hero program. It was emotion, motion, a second and public comment. CNN. Please cast your vote. Oh. Was your public comment? Yeah. Please come forward. I thank you, Mr. Mayor. Thank you. Members of the council. My name is Alex Mitchell. I represent the hero program. Just wanted to let you know that the hero program. Has been in California for the last three and a half years. We're in. Over 20. We're in 28 counties, and we've completed over 29,000 energy efficient projects to make homes. Greener and more energy efficient. And this includes anything. From solar to water. Efficiency. We've done. Almost. $550 million in home improvements. And just in water. Alone, because that is a very important, timely issue. We have saved over 5. Billion gallons of water in homes and that's equivalent to 16 million showers. If you have any questions in regards to this issue. Please let me know. Thank you so much for taking this to council. Thank you. Next item, please. Cast a vote. Oh, with the vote. Sorry, it is late. Okay, let's go and take a vote. Mm hmm. Yeah. So motion carries seven zero. Okay. Next item.",
+        "QA_pairs": [
+            [
+                "What is the first resolution the City Manager recommended to adopt?",
+                "To join the Victory Pace program."
+            ],
+            [
+                "What is the second resolution the City Manager recommended to adopt?",
+                "To join the California first program."
+            ],
+            [
+                "What is the third resolution the City Manager recommended to adopt?",
+                "Consenting to inclusion of certain properties in the California Hero program."
+            ],
+            [
+                "Who represents the Hero program?",
+                "Alex Mitchell."
+            ],
+            [
+                "How long has the Hero program been in California?",
+                "Three and a half years."
+            ],
+            [
+                "How many counties is the Hero program in?",
+                "28 counties."
+            ],
+            [
+                "How many energy efficient projects has the Hero program completed?",
+                "Over 29,000 projects."
+            ],
+            [
+                "How much has the Hero program done in home improvements?",
+                "Almost $550 million."
+            ],
+            [
+                "How much water has the Hero program saved?",
+                "Over 5 billion gallons."
+            ],
+            [
+                "What was the result of the vote on the resolutions?",
+                "The motion carries seven zero."
+            ]
+        ]
+    },
+    {
+        "idx": 3,
+        "original_prompt": "Item five, proposed ordinance 2016 0392. This is an ordinance relating to the transportation concurrency. And our our Transportation Department has done an amazing job of rewriting this, and they deserve a medal and a halo. So, Mr. Carlson, would you begin the briefing on 2006 0392? Thank you, Madam Chair. As you say, this proposed ordinance relates to the county's transportation concurrency program for the unincorporated area. It modifies the King County Code language on transportation concurrency, and it also approves a new concurrency travel shared boundary map and a new concurrency test results map. And for those who have not thought about concurrency in the past couple of years, that's a little bit complicated. So let me just give a brief outline. The and and I will say that Jay Osborne from Rhodes is here as well. And we have two members of the Transportation Concurrency Expert Review Panel. And Jay and I were planning to do the initial outline. First, there is concurrency language in the King County Comprehensive Plan, chiefly in the transportation chapter, and that sets requirements for the concurrency program. It also establishes the level of service standards for various land use areas. So for example, the rural area has a level of service for its roads, which traffic has to be more free flowing than in the urban areas. And the comp plan also requires that we do concurrency through the use of travel sheds and testing of traffic flow on arterials. I'm going to ask the committee assistant to call up our maps. That was the last handout so we can start. Under current law, we have 25 travel codes and you see the boundaries there are on the map. And in compliance with the comprehensive plan requirements, each of these travel sheds is an area where the traffic in that area uses the arterials and we test the travel speeds on the arterials. The code says every two years and then. The data is analyzed to generate a map showing travel sheds that are close to development because they fail concurrency. The way you fail concurrency is that. 15% or more of your miles on those arterials do not meet the relevant lower standard. So switch to the next. I guess I can switch to the next without. Or not. How do we get. How do we get to the next slide? Next one. Sorry. So we now have the Christchurch travel sheds are closed, there are five of the 25 and in each of those the roads and a little bit too much congestion during the afternoon peak. So the exceeded the standards and. The current proposed change in 392 makes a number of changes. It notably changes the travel shed boundaries. And if we go back to the last map, we see the new boundaries. It. Features of this map are new boundaries that reflect changes in the unincorporated area. It separates out the urban portions of the travel shields and makes them separate. And the urban unincorporated travel shelters are littered throughout. The new rural travel sheds are numbered, they are larger, and they reflect annexations. And they continue to have a logical configuration of roads that people under travel should use those arterials, and the arterial test is performed. It's not in the ordinance, but a new set of data from a local firm is used to identify the travel speeds on the arterials. And it's a much more thorough process of evaluating travel time than the old practice in which road staff actually got out and drove the roads and they had gizmos attached to their vehicles to monitor the travel times. So we have a much better picture of the actual travel times. Another change that is contained in this ordinance is under the current system, certain state routes are used in concurrency and the comp plan policy says that that may be done. It is at the discretion of the policymakers to use those routes. The the new proposal chooses not to use those state routes in the concurrency test and to stick with the county owned arterial routes. At this point, recognizing that this is a very complicated project, I'd like to suggest that Jay may come up. And if you have questions about what I've said, I've I've studied this a little too deeply, so explaining it is difficult for me. So with that, we do have a councilmember. Councilmember Dombroski has a question. Female Chair And Paul, thank you very much for your work on what seems like a little bit more of a new or a different approach to concurrency. On the last issue that you raised with the current plan, at least if I understand you correctly, permits us to include in the travel time analysis the use of state roads , which are an integral part of the world transportation system, for sure. What is the rationale for not using them? It seems like the average driver wouldn't necessarily distinguish between a state highway or state road and a county road when taking a trip. Okay, first it is certain state roads. It is not the statewide significant ones like the freeways. Right. The certain state roads that have characteristics similar to county arterials have a level of service standard that's set by. Is it the state or the Puget Sound Regional Council that, you know, the PUC sets them and so they're they are out of our control. And so the decision here was to focus on the roads that are within the county's jurisdiction and control. And in fact, the the data was gathered for the state routes that would have been used. And there is one shared that would have switched should two, but there wouldn't have actually been a difference in any other travel. Should switched from. What from open to to closed within this analysis. So in that case using the state roads because of their level of service standard will close, the travel said, meaning that development would be restricted. Now just saying no. So Jay Osborne, deputy director at roads. So the state highways in question two or two, two or three of our 900. This level of service for those is D in the rural area, but the counties level of service for roads in the rural area is B So when we test the state routes against the counties level of B, they do not pass, but they meet the state level of D and our passing the concurrency test for the state's purposes. So one of the complications in what we've been doing is to test state routes at our standard and not the states, whereas they don't meet our standard of B, they are meeting the state standard and therefore passing concurrency, which has been one of the mixes and testing state routes in the rural area. So what is the implication of this policy choice to somebody who has property and wants to do a plot to build homes. In the rural area? At the moment, using the test that we proposed, it would pass and they would have the ability to do that. And for example I've been travel said to. Councilmember Lambert's district mostly there. If you. He used the state standards and roads. He'd have like two or two up there. Kathy is that right? Mm hmm. And it might close the travel show, right? If we use the county stamp. Staff applied the county to the state. Yes, exactly. Okay. Thank you, Madam Chair. Thank you for your patience as I kind of worked through that. Yes. Councilmember, you're going to speak in detail. So but let me just put my question on the table so you can think about it. So this councilmember is used to judging concurrency at intersections. So this is sort of new to me. So it would be really helpful if you could kind of lay out how it's calculated and how you did it. But if we could just sort of reinforce that also what causes failures is some kind of average of all the roads are because if any one road is failing, that doesn't mean the whole travel shed is failing . And then just the bottom line here is if that particular change you were just discussing is made, I think the net real world impact is we're going to allow more congestion and continue to allow development. Okay. So which is I mean, which is it's about a reasonable reaction to a real world situation. But essentially, we're going to allow we're going to lower our standards for how free flowing the traffic has to be in order for development of energy to be allowed. Yeah. So there are many ways to test concurrency. So the counties methodology, we used to do actual travel time, which meant that we had staff in cars with stopwatches in the nineties driving a length of all of novelty road with a stopwatch to see how long it would take. Going back and forth between three and 6 p.m. three times in the spring when school was not was in session and not at spring break. And we would have people standing on overpasses with the stopwatch. So that's the difference. And we do the whole length of the roads. And so it's the arterial roads in that sched are all tested. And the rule is 85% of them have to meet your standard for the shed to pass concurrency. When you add state routes in because they exist in those areas and you test them at the counties level because state routes are designed to take a greater level of traffic than some of the arterial roads that we have, they don't meet the level of service. B We had a group of grad students from the U. Dub who did their MBA thesis on concurrency and found that the counties level of service B is one of the highest in the country. That set as a very aspirational level of service. So the state choosing to put D on those routes and then passing is a different standard and testing those creates complications. So to answer your question, if we added the state route testing in which we did do, and to back up for a moment about what we're doing with data, there is a firm that we were able to buy data for 24, seven for a month on those roads and then pull out the testing for every day Monday through Friday. Actually, I think we use Tuesday, Wednesday, Thursday data for those afternoons. So rather than someone driving three or six times, we had all of those data points to test out how the traffic was running with this index data. Yes. So based on cell phones. Yes. And pulling that together, which was actually more cost effective than paying staff to be out in cars on the road, testing all those areas that and we don't have that many staff to left to drive on those roads in our planning group. So one of the things that in the rural area I think is key is that letting the zoning code deal with development and how much development is actually left in the rural area is very small for the impacts on the county road system and how much development is going to put cars on the road in those open sheds. And the impact there in. Terms of climate should be thought. And this is a bigger than just the county in dealing with unincorporated areas. I would dearly love it if we could come up with some kind of a regionally consistent way of doing concurrency. And I could see it being done differently in rural areas and urban areas. But the way the city of Seattle does it, to the extent they do it at all, the way that cities like Bellevue do it and the way you all do it are all completely different. And so it's very hard for us to have a common vocabulary for the public around how well or badly the roads are doing . And then we end up essentially being driven in transportation policy by individual anecdotes of my experience behind the wheel. And I think that that's important how people are feeling about their commutes. But it would be much, much better if we had a systematic and clear way of talking about what's going on with the whole system. That's my soapbox on these things. I also dearly wish that we had a way of including throughput as part of our calculations, because it's not just how fast the individual vehicles are moving that's important. It's how many people were able to move through, you know, these points from point A to point B, if you have a really a an arterial that's really well served by transit, even if it's going slower, it could be moving lots more people than one that is really not well-served by transit. So anyway, thank you for listening. Maybe next time we can work with our colleagues on trying to bring some of this stuff together, but I appreciate the work you've done here. I think it makes good sense given the realities of what we're dealing with, especially splitting out the urban from the rural makes good sense. Thank you. So, Councilmember, if you had been here years ago when we had our old currency plan that had, what, 300 and how many, 360 boxes? It was a nightmare. And we hired a national firm to come in to to give us some feedback. And there was the worst plan they'd ever seen in the country. This man said he had like 30 years experience and he'd never seen anything like it. So he pretty much ditched that afterwards. And so this is the new improved, the new improved, the new improved. And we don't have any jurisdiction, as you said, over Bellevue now that you're not mayor anymore. So we really can't unless it goes through, you know, your transportation committee. He has RC to, you know, to make those changes. But I think the thing that's really important is that people are driving from the rural area into the unincorporated incorporated areas and there's one level of service out here and then there's another level of service in here. The drivers driving, they know they're going to be in commute traffic. And so to have one level of service that is asked for, okay, that is aspirational so that, you know, it's an artificial barrier. And I don't know if it's. Thank you for handling that out there. I don't know. Is it still true that on the urban growth boundary line that the the part of the road from the Senate line to the rural area has one level of service and from the center line to the urban line is another level service, or do we fix that? A couple of years ago, I can't remember if we did or not. We did fixed fixed it. Okay. So that used to be a problem that the same road could have two levels of service on it. So we did fix that. So that's good. So I think that this makes it easier. It's more consistent with other roads in the county. And the other point I think is really important is that there isn't much development left in this county other than what's already been delineated under the Growth Management Act. So we know what that's going to be. So I think this, as Jay said, is an ability to deal with the roads that we have control over. And if I may make one final thing, thank you. And then I promise that will be be. And when I look at this map and I see the circles inside the travel sheds, those are the urban islands in the unincorporated areas. I think there's been a lot of talk and consternation about the growth targets in those areas. And I have to say that this map demonstrates part of the reason why there's a debate and why it isn't just a one sided. We need to grow. We're growing. Let us grow. The other side of that coin is the more we allow or encourage or support large amounts of growth out in those urban areas, the more you're going to see these travel shed suffer because they have to serve and and support growth between there and end. We are still requiring a more free flowing state of traffic for there to be ongoing development than we are in the inner suburbs of the urban areas. So it's a this sort of demonstrates one of the complexities of that whole debate. And, you know, it gets into a lot of the debates with we're talking about affordable housing when we're talking about, you know, certain kinds of lifestyles in the rural area where the cities, which is where people are supposed to be, you know, growing out there. And it's a housing choice that for some parts of the county, there are very different housing choices in different parts of the county. So it is a complicated issue indeed. Okay. Did you want to continue? Well, I was going to say on page 46 of your packet is the actual list of the root segments that failed in this analysis. And those are miniatures which travel should they're in, as you said, the total mileage within the travel shed. If 85% or more passes, that's the test. If less than 85% of the mileage fails, then the travel should is failing and only one travel should fails in this new process. The the other point that Jay alluded to in terms of development in the rural areas, even if the travel should fail, there are there is provision for minor and certain public and certain educational developments to proceed. And our concurrency system has always allowed that the form in which that has been authorized has changed. And so a section of the code that is amended in this proposed ordinance for. Ten 7285. Lists those minor developments and schools and other uses that can still go forward if a travel should fails. And that's particularly important, for example, because one of the old issues that we heard a lot about was a family that had owned a parcel that wanted to subdivide so the children could build a house. And that's something that the county has modified the program to accommodate that kind of use as long as it complies with the zoning. And, you know, again, concurrency is the first step in developing something. You have to be consistent with your zoning as well. So I think we should show the last map which shows the results. That's the point. Yeah. This is so this is the test results map. And the red arrow there shows the one close development the close showed, which is mostly agricultural production district and is it does not have a lot of areas that could be developed anyway . Oh, that's interesting. It is APD and the parcels, there are probably minimum ten acres, so you're not going to be getting a whole lot of traffic out there. So how did that end up getting closed? It's okay. On page 46, that's seven and there are two. It's a small shed with a small mileage and there are two road segments, each a half mile long that fail. And that puts it over the top. It's. It's an odd area because of the agricultural uses. So to 72nd to 77, that's the main drag across the valley goes through the APD, which is four lanes going through there at an urban level and it is being tested at a level of service fee because the ag area is rural. So it's the urban road going through there, being tested at the rural level, which is why it fails. So if it's an urban road, why are we testing it, the rural area, just because it's in the APD? All right. Okay. So what? Oh, council member and about. Sorry. Thanks. I think when we looked at the concurrency issues a year or so ago, we kind of parked them because there were some open issues and the testing had gone and done this work. And one of the issues at the time and this kind of falls on concern about duties, regional consistency, if you will, but on a more narrow basis. And that was I was interested in travel sheds that cross the urban rural line. And at the time we had something like up the East Renton Plateau and and there was then a question about whose standard should you apply? Right. And it seemed to me it made sense to at least take a look at the adjacent city standard and the urban side of the line along the line of thinking that we should account for, you know, the city's planning policies and zoning traffic standards, that kind of thing. So my one takeaway that I'm getting from this, I'd like to make sure everything is it's a move to not cross the urban rural line in travel sheds. We've now got travel sheds on the rail centerline line where we can have one set of standards and then on the urban center line where a different step might have apply some nods there. That's that looks good to me. And the related question then is within the urban side of the line, will we do we in this proposal or will we in the future start taking a look at the city, you know, the city that has the paid for their standards and incorporating that into our into our level of service standards. So in the history of concurrency, we've had agreements with various cities to do that and to do that development. When the economy suffered in 2008, there were four cities that withdrew from those agreements looking to be able to develop their areas and what they needed to deal with, because there's also an impact to the mitigation payment system and how much money that you were getting for development. It proved to be somewhat complicated as they went forward. And we have had conversations with some cities about those standards and those areas continue to annex Kahani and some Amish being an example and is acquire Fall City Road, some issues that they were interested in developing. But currently we don't have any agreements to model concurrency in those areas, in part because the remaining urban areas are quite small now and have been chipped away at. So I think it's important to know that we as a county in the state have the smallest amount of unincorporated area of any of the county. We have 12 and a half percent of the county that's unincorporated. Snohomish County has 28% and everybody else is in the forties and above. So we have done what the Growth Management Act said and again Incorporated. So I'm talking about 12%. Yeah, I'm talking in rural. Urban or rural. Unincorporated. And I'm just looking at the map here. I don't think I don't think 12% of the county is. An anchor, but. I think at least landmass, it's it's quite a bit bigger, maybe population, it's probably about 300,000 out of two. It's about 250 something. So anyway, that and I don't remember exactly how they calculated that 12 and a half percent, but we are far lower than everybody else. And so the land has already been allocated to whatever it is, one acre, five acres, ten acres, 20 acre parcels. So I think that as we go forward and we look at that, we have an aspirational level of B and then the people get off the B road and they get into a city and it may have a D or an F rating. The dichotomy of being and on this part and going this half mile at a B and then this half mile at at F or this half mile, the D, I think that as as we look at this, we need to be more realistic about how high that level is and making sure that, you know, people who own land or would like to have their properties, the device, the children could live on the property and take care of them. That. We make allocations for that. So. Okay. So what is the the will of the body? We need to have a 30 day period for putting this out for when they call it public testimony. 30 day advertising period. So would you like to vote this out of committee with or without recommendation? What would you like to do? Well, there's an amendment that has that been described as looking like mostly technical cleanup. Or is there some policy changes there? Well, yes and no. Yes. Yes. Okay. Yeah. You have before you Amendment one a which is very slightly different from the amendment in your packet. It I would say that, yes, it makes mostly technical changes. There were a couple of spelling errors. There is a new sentence added to section eight. The first. And this is. How does that. No, no, we're not at all yet. How did it get up to be, anyway? It went in front of me. Yeah. The section four of the ordinance there is the online nine amendment when it says except except as provided in KCC 1470 285. That's I would call that a technical clarification that a minor use is covered by 285, which is which has always been the case. So it's not changing any practice when ten is a typographical change. Then starting on the line 13 Section 1470 285 L This is the last item on the list of those minor developments that are allowed in feeling travel sheds. And this is there's some rewording for clarity. And then down on line 18, it says the property has not been subdivided in the last ten years. This relates to a short subdivision in a rural travel shed where the owner wants to subdivide. And this is the classic family method of requests. And under current law, if the applicant has owned the property for five or more years and the property has not been subdivided in the last ten years, then that's allowed if it meets the zoning requirements and there is no need to purchase transferrable development rights as part of that deal. This is this is how it has been. The executive transmitted proposal was going to change the no subdivision in ten years requirement to no subdivision in five years. And in reviewing this, we found that there is a rural policy are 3 to 3 in the just approved plan that says ten years is the requirement. So we're we are maintaining the existing language for ten years and not moving forward with the change that the executive has proposed. And it is simply because comprehensive policy language is mandatory on that point. Can I ask the question? Is the executive okay with the revised amendment? Yes. Okay. So I would have preferred the old, but the new was what we just passed out. So I will tell you that when the plan comes up again in four years, that I would like to reconsider this. But but anyway, that's the way it is at this point. Councilmember Balducci. No, my questions were answered. Thank you. Okay. Councilmember Dunn. Thank you, Madam Chair. Just a concurrences, an issue I've worked on for a long, long time. What? What? You know, it's fairly not well understood by most elected officials. I fear you're changing the slightly modifying and expanding the travel sheds, but you're not changing the methodology methodology for the actual concurrency standards in this. Right. Is that correct? The the methodology the in the in the service. Level, E for example, those. Sorts of things. You know, the the comprehensive plan establishes the level of service standards for urban as E, rural as B and then there are the rural town centers are D and rural no are E and rural neighborhood centers are D. You did not change those level of service standards in this latest update of the comp plan. So they still remain in place. Okay. And and we used to use a red, yellow and green map for concurrence. You remember that? And that's gone. Is that no longer what we're using? We're going to modify to this this mapping. That that was when when we had those hundreds of individual zones. And at one point it was written red and green only that it was red, yellow and green. When we moved to the Travel Shed concept a few years ago, 2008, I think it was the colors were abandoned. Okay. You know, I generally, maybe more than most up here, I tend to believe we need to. Be building homes. Condos, low income housing, what have you, because we need to put places for young families to live and for everyone to live. And so I'm with what you might call pro-development, but we've got a situation that's developing in earmuffs. Right? I know pro-development. I know it's bad. I didn't win. So. Yeah. And and so the question I have, I'm looking at south of Issaquah, you know, the is for Hobart Road, a road that is so bad that I pretend it doesn't exist because you will get lost in the vortex of traffic forever and they are deeply unhappy citizens there. People can't get in and out. Emergency services can't get in and out. Ambulances, fire trucks. It's it's awful, largely because this county refuses to increase capacity on the road. That's another issue. But I'm not seeing something here that's precluded development in that travel shed. What's the status of the Esquire open road travel should I think I saw was number 12. Well, no, you're on spot here. So if you want to pass it on to some of your colleagues. Well, the the crude travel schedules for the new travel shed would also be open and. There is a segment of Issaquah Hobart Road that feels it's between the Issaquah City Limits and Southeast 127th Street. So. So there wouldn't be development wouldn't be permitted to make a long story short in that section. No, it's it's the total results for the travel should in within within travel should for you do not hit the 15% or more mileage feeling standards. Okay, that's it. So I'm almost done. Madam Chair, I appreciate that. Okay. So I've never believed that currency ought to be the way to control our land use planning and development. I think that's the wrong way to do it. I think we ought to be doing it through zoning, through other permitting related issues. But we've got a. Real problem with this for Hobart Road. Part of it is a willingness to increase capacity. A bigger part of it probably is the fact we have the money to increase capacity. Maybe it's a little bit of both, but I just want to point out that. If you put large developments, even if they're an RFI of zoning out there, you are just going to add to a problem that is already disastrous. It's more of a statement than a question. And so. Makes me wonder whether these broader travel sheds are really the right way to go. As a matter of policy, I'm not going to object against it, but I'd like to drill down on it further in the future. So since I've been here, we've had several different renditions of what the concurrency looks like, and it's gone from absolutely obnoxious to be figured out. You need like a Ph.D., which is what the expert came in and said to something more simplified. The amount of growth going on out there is is very small compared to what it used to be. So I want to clarify. I've gotten some clarification. 12% of the population in this county is in the unincorporated area. So it is by population 12% of the population. Half of that is in areas that can be annexed. And so 6% is in the rural unincorporated. So there's not a lot out there. So we have some people with us that need to comment too and had some really important things to do that have studied this. So do you want to make some comments also? So just as a quick introduction, okay. In this, the council a number of years ago appointed a Transportation Concurrency Expert Review panel to review the work and provide a comment letter on every thing that was submitted to the council as we went. And it's represented from folks from the development community, the environmental community. We have a citizen of the unincorporated area and we have a representative of the Non-Motorized users and bus and transit as well on that. In this legislation, the Transportation Concurrency Expert Review Panel has decided that their time has come to an end and that the methodology and the amount of development and what we're doing with concurrency is something that they support. And the 1:00 scholar who's to my right, who is the chair of the Transportation Concurrency Expert Review Panel, it's going to give you a few remarks. Good morning, council members and thank you very much for your time. I would also like to let you know that one of our very long standing members and Martin is here in the back and I think her attendance, in addition to mine, I hope, conveys a reflection that this panel had quite a committed and long standing involvement with staff as both historically in terms of the older approaches to concurrency as well as the approach that we're putting forth to you now in these materials. Probably the most significant aspect of the panel that I, I was influenced by was both the collaboration among extremely diverse interests, as well as what we're giving to you today, which is essentially a unanimous recommendation, despite the broad diversity of interests on the panel. This panel has worked together for many years. I am the most recent addition to the panel, although I, through my former former colleague Bob Jones, knew a lot about what was going on in the background and am extremely honored to have taken over chairmanship of the panel a couple of years ago. And we're pretty proud of well, I should say we're extremely proud of the work that our are very well-educated staff has done on this, as well as the master's program materials that were presented to us in the past year. One of the things that I think is pretty interesting is that staff has been pretty selfless in this process. They were very interested in the good of the county and the good of the system above all. And in looking at that data, I think that is really reflective of some very thoughtful work that's been given to you . So and I would say that all of us on the panel again felt that we were extremely well served, not just by our consensus, but by a very well-educated group of individuals who could really so succinctly convey information to us to allow us to have a pretty candid and often very spirited dialog, as you might imagine, considering the members on the panel. But ultimately, we feel good about very good about what we've presented. We're sad about dissolving because it's one of those few fora where we actually get to get together and talk candidly without having to put other people's interests on the line, but really have good quality conversations. But it makes a lot of sense at this point to dissolve. And so we're very honored to have served the county. We thank you very much for the opportunity, and I hope that we can continue to be of service in our individual capacities. First of all, I'd like to thank you, as it's been said a number of times, if here this is a very complicated formula, it's very impactful. And so having somebody willing to sit down and look at all this and and bring a unanimous decision back is very much appreciated. And we thank you for your service. So essentially what I'm hearing you say is that you believe some tell me this is right, that you believe that with the lack of growth happening right now, that there's no need for you to continue on as a committee to evaluate this. The panel believes that both because of the way that the travel sheds have been reformatted and the annexation processes that are going on, as well as the ability to use a lot of that more mechanized methodology through INRIX, that there just isn't a need for this panel to both take their time to review these aspects that, yes , are becoming a little bit more rote in their processing. And we don't need to take staff time to be putting together materials when we don't necessarily have a deliverable we may need to bring to you. I don't know what the future holds, but for the moment, we're comfortable with the decision. Excellent. Okay. And I'd also like to thank you for being so cognizant of other assets like INRIX. And I know we use it at other committees and the data has been very, very helpful. So thank you for seeking that out to you. Okay. So now I'd like somebody to put this before US Council member. But did you manage to move approval of proposed ordinance number 2016? Dash 0392 of the do pass recommendation. Thank you, ma'am. Any questions or comments before we take the vote? Okay. Councilmember and Ambassador. I'd like to offer Amendment one. Oh, yes. Thank you very much. Yeah, I did have a question before we and we can vote on that, but just got a final. Okay. Thank you. I think this has been well explained by our staff. It isn't exact, as my name's on it. When I'm speaking to it, I would prefer that some of this wasn't changed this way. The correctional errors and the typos and stuff. That's great in the clarifications in the King County code, that's fine. It is the five and ten year issue that that does bother me. But because we just passed the comp plan that was voluminous and somehow that was the change in there, I think we need to flag that for three years and ten months from now and maybe change it back. But at this point, having all of our code be consistent is probably a good thing. So all those in favor of when a as presented by our staff please say I as opposed name is passed and now before us we have the amended version of 2016 0392. And Council Member Dombroski has the comment. Just to make this a follow up to Councilmember Dunn's inquiry about capacity and related funding. And when somebody does a project and they may pay some mitigation money, right? Does that money under our current provision, does it need to be spent within the travel shed where the projects occurring? So in the current provisions, it's SIPA money that they're actually spending on the roadway. And so it's for specific projects and identified for those within the travel. Within the travel said, okay, thank you, thank you, thank you. Okay, thank you. That was a good clarification question. I'm glad you asked that. Okay. Are those in favor of call for the vote from the clerk's office? Councilmember Baldacci. Councilmember Then back. Councilmember. Then I remember. Gossage. Councilmember Colwell. Councilmember McDermott. Councilmember of the group. All right. That's number one right there. Madam Chair, I mean, I'm sure the vote is six days, zero no's and councilmembers, Gossett, McDermott and moderate. They were excused. Okay. So. Do we want this on consent or do we want to talk about it again? What would you like? Didn't I hear you say that it needs to be put out for Thursday? Public comment. Oh, that's right. That's right. Says that's not enough for sure. Okay, that's good. Thanks for pointing that out. Okay. And it does take a 30 day advertising period which can start. So this will not be on the regular schedule because we have to wait for the after the 30 days, which will be the end of February. So if there is no other business to come before this committee, the meeting is adjourned. Thank you, everybody.",
+        "QA_pairs": [
+            [
+                "What is the proposed ordinance 2016 0392 about?",
+                "It's about transportation concurrency."
+            ],
+            [
+                "Who is Mr. Carlson?",
+                "He is the one briefing on the ordinance."
+            ],
+            [
+                "What does the proposed ordinance modify?",
+                "It modifies the King County Code language on transportation concurrency."
+            ],
+            [
+                "What does the concurrency test results map show?",
+                "It shows travel sheds that are close to development because they fail concurrency."
+            ],
+            [
+                "How does a travel shed fail concurrency?",
+                "If 15% or more of your miles on those arterials do not meet the relevant lower standard."
+            ],
+            [
+                "What does the current proposed change in 392 do?",
+                "It changes the travel shed boundaries."
+            ],
+            [
+                "What is the new proposal choosing not to use?",
+                "It chooses not to use state routes in the concurrency test."
+            ],
+            [
+                "What does the ordinance contain?",
+                "It contains changes to the current system and approves a new concurrency travel shared boundary map."
+            ],
+            [
+                "What is the implication of the policy choice to someone who wants to build homes in the rural area?",
+                "They would have the ability to do that."
+            ],
+            [
+                "What is the status of the Issaquah Hobart Road travel shed?",
+                "It would be open for development."
+            ]
+        ]
+    },
+    {
+        "idx": 4,
+        "original_prompt": "Very good. Any comments? Those in favor of confirming the appointment. Please vote i. I. Those opposed vote no. The motion carries and the appointment is confirmed. Please read the part of the Civil Rights, Utilities, Economic Development and Arts Committee. The report of the Civil Rights, Utilities, Economic Development and Arts Committee. Agenda Item six Council Bill 119169 An ordinance relating to the Department of Parks and Recreation authorizing the acquisition of real property, commonly known as 50 104 Southwest Orleans Street, and authorizing acceptance and reporting of the deed for open space, park and recreation purposes. The committee recommends the bill pass. Because I'm a herbold. Thank you. So this is something that is, as we've heard, a long time in the making. Former Councilmember Tom Rasmussen worked on this issue for a couple of years before I joined the council. In fact, my first visit to this particular property was before I took office in mid-December. The tour that was organized by Councilmember Rasmussen staff member also included the Southwest Historic Society and it included for Terra because at the time the Parks Department was not interested in pursuing the purchase of the property, but for Terra was interested in perhaps offering an interim solution and holding on to the property for a period of time and perhaps at a later date transferring it to parks. Happily, the Parks Department, through the persistence of many, many people who've joined us today, changed their mind. And the thanks goes to many people both inside and outside City Hall. I want to thank, first and foremost, Bruce Statler. I think, you know, he said it best when I visited him at his home when he realized the value of the property to the public and in making what is truly more of a donation than a property acquisition and in keeping in tradition with the history of the park , which was initially a donation in 40 I'm sorry, 1908 by Ferdinand Schmitz for the enjoyment of the public. And, you know, Mr. Sattler is offering this piece of property to the city for well under half of its value. And he did so because he realized what the future would hold for that piece of property should he sell it. And he was very concerned that the property would be be redeveloped for use as a as a McMansion, thus depriving the general public from the enjoyment of the park. And not only was the is the acquisition helping stop something that is not desirable for for that park entrance, but it actually is adding something, I think that's really important to future generations for enjoyment of the park and potentially looking at a new access trail in the future so that there will be more ways for more people to to enjoy the park. I also want to thank Councilmember Rasmussen for his persistence. I've kind of seen this as one of several legacy projects that the councilmember has had, and I've really enjoyed working with you. It's been a great help to me in pushing this forward to to point to to your efforts and your commitment on this throughout the process. I also want to want to thank for Tara. Thank thank Vicki Schmitz and thank the Southwest Historical Association. And in particular, we've got Jeff McCord here with us today. But your predecessor, Clay Eales, was a great advocate for this project as well. Many thanks as well to park superintendent Hazel Segarra, Anthony RMR Parks, staff Chip Nevins, Tracey Ratcliffe and Mike Fong, who was in the mayor's office at the time and is back in the mayor's office again. Those folks all helped a great deal. The life estate option will allow Mr. Statler to continue to live in his home, and in return the city will acquire the land at a reduced rate and at that time in the future will become officially part of Niche Park. Thank you. Very good. Any further comments or questions can send back show. Thank you. I just want to acknowledge Councilmember Rasmussen, former councilmember said like working with you and I'm glad you're back. Bruce, thank you for your generosity. And I loved the article about you yesterday front page of Seattle Times and to read about Scout and Nellie and I'm sure that they'll continue to have a great home. And this park I think it's 53 acres. It's a stunning pedestrian area in the middle of West Seattle. And you're contributing more to that. Thank you very much. Very good. Okay. We're ready to vote. Please call the roll on the passage of the bill. Gonzales i. Herbold, i. Johnson Whereas macheda i. O'BRIEN So want to make sure. President Harrell hi. Nine in favor and unopposed. Bill passenger would sign it. Very good. The next such an item and the short title. Agenda Item seven Council Bill 119140. An ordinance authorizing the general manager and chief executive officer of Seattle Public Utilities to enter into agreements with the Port of Seattle and BP West Coast Products, LLC for the purposes of satisfying utility related conditions for the Port Street vacation petition for its Terminal 18 redevelopment project.",
+        "QA_pairs": [
+            [
+                "What was the agenda item six about?",
+                "It was about authorizing the acquisition of real property for park and recreation purposes."
+            ],
+            [
+                "Who worked on this issue before the current council member?",
+                "Former Councilmember Tom Rasmussen."
+            ],
+            [
+                "Who was interested in offering an interim solution for the property?",
+                "For Terra."
+            ],
+            [
+                "Who is donating the property to the city?",
+                "Bruce Statler."
+            ],
+            [
+                "What was the initial concern about the property?",
+                "It would be redeveloped for use as a McMansion."
+            ],
+            [
+                "Who else was thanked for their help in the process?",
+                "Councilmember Rasmussen, Vicki Schmitz, Southwest Historical Association, Jeff McCord, Hazel Segarra, Anthony RMR Parks, staff Chip Nevins, Tracey Ratcliffe and Mike Fong."
+            ],
+            [
+                "What will the life estate option allow?",
+                "It will allow Mr. Statler to continue to live in his home."
+            ],
+            [
+                "What was the result of the vote on the bill?",
+                "Nine in favor and unopposed."
+            ],
+            [
+                "What is the size of the park?",
+                "It's 53 acres."
+            ],
+            [
+                "What is agenda item seven about?",
+                "It's about authorizing agreements with the Port of Seattle and BP West Coast Products, LLC for the Port Street vacation petition."
+            ]
+        ]
+    },
+    {
+        "idx": 5,
+        "original_prompt": "Thank you very much. Congratulations. We have and again, because we had the budget hearing, everything is just taking longer than it normally would. We have one more hearing tonight and that's hearing for or the third hearing on the agenda, and then we'll go into the regular agenda. So this is hearing item number four, which is an early vacancy. So, madam, please read the item. Report from Public Works recommendation to receive supporting documentation into the record, conclude the public hearing. Find that the area to be vacated is not needed for present or prospective public use and adopt resolution ordering the vacation of the north south alley west of Long Beach Boulevard between East Waldo Road and 35th Street, and a portion of sidewalk right of way along Locust Avenue, District seven. Thank you, Mr. Modica. That report would be given by Craig Beck, our public works director. It's members of the council. I think we did a really good job of describing what we're proposing in the recommended action. The staff have been working closely with laser fish to help them expand their footprint here in the city. They're looking to add more office space for their operations. They own two parcels where an alley cuts those parcels. And we're looking to vacate that alley to allow for the development. To move forward. That concludes my report and I'm available for questions. Thank you. I appreciate that. Let me since we're this is a hearing, let me go ahead and just continue. I want let me go out and do public comment first for the hearing. So for the Ali vacation. Carelessly, Robert Fox, Mr. Avaya and Jeff Miller, please come forward now. Mr. Miller. No. Okay. So concludes public comment. Let me go ahead and go back to Councilman Ringo. Thank you. I think this is a great project. It's opportunity for Lady Fish to complete its its expansion that it's doing to maintain their business leader fish. It's a great corporate partner here for Long Beach, and this is a great project that will help them expand. And I would appreciate the support of my colleagues. Thank you, Councilmember Austin. I'm just going to I want to just add that this is a fantastic project. This Ali vacation will allow us to expand and support the incredible work of Lazar Fish, which is an early tech company within the city of Long Beach that took a chance on Long Beach. And I always want to recognize that they went in, took a chance. Now they are growing by hundreds and hundreds of jobs on their campus and creating a model facility in Bixby Knolls as a gateway. They're clearing oil fields are replacing fences, they're doing landscaping, they're fixing alleys. And they're the exact example of the type of corporate partner you would want in your community. And so I want to thank Mr. Wacker and his entire team. They're really doing a great job and thank the councilmember for his incredible support. And this project would not be where it's at if it wasn't for his support as well. And with that, there's in motion any second. So let me please ask you to cast their votes. Councilmember Richardson. That's why Richardson motion carries. Thank you. We're moving on now to public comment and consent. I made you consent calendar first. Can I get a motion any second on the consent calendar, please? I have carelessly Robert fox carlos over here on the consent calendar.",
+        "QA_pairs": [
+            [
+                "How many hearings were scheduled for the meeting?",
+                "One more hearing."
+            ],
+            [
+                "What was the fourth hearing item about?",
+                "An early vacancy."
+            ],
+            [
+                "Who was to give the report for the fourth hearing item?",
+                "Craig Beck."
+            ],
+            [
+                "What is the purpose of the alley vacation?",
+                "To expand office space."
+            ],
+            [
+                "Who owns the parcels where the alley cuts through?",
+                "Laser Fish."
+            ],
+            [
+                "Who was invited for public comment?",
+                "Robert Fox, Mr. Avaya and Jeff Miller."
+            ],
+            [
+                "Who spoke in support of the project?",
+                "Councilman Ringo and Councilmember Austin."
+            ],
+            [
+                "What company is the project supporting?",
+                "Lazar Fish."
+            ],
+            [
+                "What are some of the improvements Lazar Fish is making?",
+                "Clearing oil fields, landscaping, fixing alleys."
+            ],
+            [
+                "Who motioned for the project?",
+                "Councilmember Richardson."
+            ]
+        ]
+    },
+    {
+        "idx": 6,
+        "original_prompt": "All right. Thank you. Thank you so much for that. And then, councilor, we're going to slide. Right into. My item. And not my item are item 11, which are my nominations to the Commission on Persons with Disabilities. And I am happy to report that I have three impressive nominees and I just want to give shout out my appreciation to our community. I am just overwhelmed by the amount, the quality, the quantity and quality of applications. I'm getting to all of our boards and commissions. These decisions aren't easy. I'm really touched by people's commitment. They just desire to serve their community, to lend their time and talents. So what I'm doing, my recommendations are that we reappoint two of the incumbents and then appoint a new individual. And so whenever incumbents is Lisa Hall Lisa has a wide range of experience with disability issues, including from a personal basis, from working with a parent. She has done a lot of work with our less privileged individuals for many years, found Christ Episcopal Church as food bank program and does work with our Rantoul community and she has served a term on the commission with persons in just with a commission on persons with disabilities would like to serve in others. She is one of my nominees and the next nominee is us. And this is in no particular order. It's in the order in which they were interviewed. Jennifer Rohloff is another incumbent and Jennifer has a brother. Her younger brother has special needs, lives in a group home in the area. But Jennifer, from a young age, in part because of her commitment, in part because her mother was a teacher in the school district, she was actually called upon and agreed to be a tutor in special ed classes when she was in elementary school. She remembers tutoring in Neil Tam's special ed class, and Mr. Tam is a revered educator. Then the late Neil Tam, his friends and family are still in the that community, but he was a special ed teacher before he became a principal. Jennifer has served on the board of class in house, which trains and provides employees with special needs folks that serve you at Safeway and other other establishments. And she's said she is a very passionate advocate for persons with disabilities. And then my third and newbie appointee is named Katie Beeler. And Katie is well, I'll just give you a little, little background. She actually applied to serve on the library board, but I was so, so taken with the description she put in her her application that I reached out and said, would you possibly consider serving on the Commission of Persons with Disabilities? Arranged for her to meet with Sarah Henry, our public information officer, who also staffs that commission. And so she's Katie Bieler, who works in the publishing industry. In particular, she is project manager on global literacy and education for a company, an independent publishers group, where she oversees publications for persons with reading challenges , whether they're visually impaired, have dyslexia, ADHD. And she says that after 15 years in the publishing industry, my passion for equitable access to information led me to my current role, where I create software products for people with print reading disabilities. She's currently the president of the Bay Area Women in Publishing. She also has a master's degree in Jewish art and in visual culture from the. Jewish Theological Seminary of America, as well as bachelor's degree in History of Art from University of Michigan and another and another bachelor's degree in anthropology from University of Michigan. And she's excited to join this commission. So that's just an introduction. At our next meeting, we'll have a chance to vote on their nominations, but I just wanted to introduce them to you. Thank you, all of you, everyone who applied and those who were nominated. And we look forward to voting on your nominations next week. So thank you very much. And now we will move on to item three. And item three is proclamation, special orders and announcements. And there are two proclamations, one I'm going to read this evening. The other one will will be posted on the website and go in the record. But I wanted to read our proclamation recognizing declaring September 15 through October 15, 2021 to be National Hispanic Heritage Month. So here's our proclamation.",
+        "QA_pairs": [
+            [
+                "How many nominees are there for the Commission on Persons with Disabilities?",
+                "There are three nominees."
+            ],
+            [
+                "Who are the incumbents that are being reappointed?",
+                "The incumbents are Lisa Hall and Jennifer Rohloff."
+            ],
+            [
+                "What is Lisa Hall's experience with disability issues?",
+                "Lisa Hall has personal experience and has worked with less privileged individuals."
+            ],
+            [
+                "What is Jennifer Rohloff's connection to special needs?",
+                "Jennifer Rohloff's younger brother has special needs."
+            ],
+            [
+                "Who is the new appointee to the Commission?",
+                "The new appointee is Katie Beeler."
+            ],
+            [
+                "What is Katie Beeler's professional background?",
+                "Katie Beeler works in the publishing industry."
+            ],
+            [
+                "What is Katie Beeler's role in the publishing industry?",
+                "She is a project manager on global literacy and education."
+            ],
+            [
+                "What is Katie Beeler's educational background?",
+                "She has a master's degree in Jewish Art and Visual Culture and bachelor's degrees in History of Art and Anthropology."
+            ],
+            [
+                "When will the council vote on the nominations?",
+                "At their next meeting."
+            ],
+            [
+                "What is item three on the agenda?",
+                "Item three is proclamation, special orders and announcements."
+            ]
+        ]
+    },
+    {
+        "idx": 7,
+        "original_prompt": "Okay. Great question, Kerry. Then moving on to 39, please. Report from economic development. Recommendation to execute all necessary documents to amend the EDA Revolving Loan Fund to create the Long Beach Emergency Microenterprise Loan Program to assist local microbusinesses impacted by COVID 19 citywide. Okay. This is a report from staff. Yes. Jon Kate Blair, director of economic development, will give the staff report. Yeah. Good evening, Mayor and members of the city council. This is a really good, positive change. In response to the current emergency, we've operated a revolving loan fund program since the Los Angeles riots of 91. We established that fund in partnership with the Economic Development Administration, a federal agency with an initial $1 million grant. Over the last 30 years, we've made $10 million worth of loans off of that initial $1 million grant for this particular emergency. We've asked the Economic Development Administration to change this program and in the terms to allow us to make a much smaller $10,000 and less loans. We've also changed some of the terms for repayment so that small businesses will have up to seven years to repay these $10,000 loans. This allows the city council to make a lot more loans to local small businesses with five employees or less. Typically, these are the businesses that are unable to secure SBA loans. The bigger loan programs that you've been hearing about and fills an important gap for both for profit and nonprofit micro businesses in the city of Long Beach . So we're asking for your approval to modify what's called the Economic Development Administration Plan. This will occur during the period of the emergency. After this funding is loaned out, of course, we'll seek repayment. We'll provide technical assistance to the borrowers and hope that we don't experience a lot of loan losses and see the revolving loan fund be capitalized at the end of the term. So with that, I'll end my report and happy to answer any questions. Thank you, Governor Richardson. Thank you, Mr. Mayor. And I want to start by thanking John. We have had a lot of conversation about this, but I think it's important that as we first of all, as we look at the different sectors of response and there's a lot of support, federal response for, you know, credit worthy businesses and this is already happening that these are, in fact, the things that really make a difference. This action is also in alignment with the city council economic relief package this week. One of the items that we asked to come back from the road to see just how we would quickly we understand the economic impact that that's taking place on our businesses. And I think that the way that we're approaching this as Microsoft flight mode makes a lot of sense, either by five employees or fewer. And even we think we've really been be concerned about the hard time that the folks to deal with are readily available messengers of many of our vulnerable communities, as you know, from from the beta, you know, and what we've been doing. Those are the people we need working with. And they're all having a very difficult time. By late, House staff will work to find ways to make this program more accessible, more flexible and more responsive. I like that there's no theme. I like the image, the format. I love the cap on interest rate. That's one half percent. All of these modifications make sure that we remove barriers to make sure that makes sure that this actually ends up in the hands of the businesses and not the. And I'm happy to make this motion. Thank you. Customers. And they have. Thank you. And there I would like to second this motion. I, I absolutely love the idea of converting an existing and successful program to meet the demands of this crisis that we're in right now. So I'm in full support of this program. Thank you for stopping to start placing your office, for being innovative and quick thinking and and remembering those businesses that are not contracted by our SBA loans. So thank you. And I'm happy to step into of them. Thank you. Councilman Austin. Thank you. I'm happy to support with as you know, there are many small businesses in my district and throughout the city that are really, really hurting at this point. And this type of initiatives actually brings, I think, help and hope for those businesses that are that are really taking it hard with this effort, home shutdown that we're experiencing. And so I'm happy to say that this. Thank you. And Councilmember Pearce. Theme, happy to support, happy to give props to the Economic Development Department. And John, I know that we had a good conversation about this on Friday. And I just want to highlight, you know, whenever people are applying for these grants at the state level or the federal level, it has been so frustrating for so many of our business owners. But the feedback I've gotten from the Second District businesses that have applied and been approved for grants or loans through the study is about how helpful our staff has been, how quickly they got a response. And so the more that the city is able to be nimble and provide these opportunities for our constituents, I think the better. You know, I think there's a definite role for for the federal government to play. But really in Long Beach, what I've heard is just that we are really setting the model for how to be nimble and how to make sure that we're doing things in a way that makes sense to the everyday user. And so, again, just really proud of the work that you guys are doing during this difficult time. So I'm happy to vote yes. Thank you. And that concludes public comment. I'm sorry, I can't comment on this item. So, Madam Clerk, roll call vote, please. District one. Hi. District two. I. District three. High District four. By. District five. I. District six. By District seven. So District eight. High District nine, high ocean carries.",
+        "QA_pairs": [
+            [
+                "Who is the director of economic development?",
+                "Jon Kate Blair."
+            ],
+            [
+                "What is the purpose of the Long Beach Emergency Microenterprise Loan Program?",
+                "To assist local microbusinesses impacted by COVID 19."
+            ],
+            [
+                "When was the revolving loan fund program established?",
+                "During the Los Angeles riots of 91."
+            ],
+            [
+                "How much was the initial grant for the fund?",
+                "$1 million."
+            ],
+            [
+                "How much in loans has been made from the initial grant over the last 30 years?",
+                "$10 million."
+            ],
+            [
+                "What is the proposed change to the Economic Development Administration Plan?",
+                "To allow smaller $10,000 and less loans."
+            ],
+            [
+                "How long will small businesses have to repay these $10,000 loans?",
+                "Up to seven years."
+            ],
+            [
+                "What type of businesses are these loans aimed at?",
+                "Local small businesses with five employees or less."
+            ],
+            [
+                "What is the cap on the interest rate for these loans?",
+                "One half percent."
+            ],
+            [
+                "Who seconded the motion to support the program?",
+                "Councilman Austin."
+            ]
+        ]
+    },
+    {
+        "idx": 8,
+        "original_prompt": "Okay. Thank you. Next step is we're going to do item number, is it that was 16. So I could do item 16. We'll try to get through these as expeditiously as possible. And there's going to be a a motion that's ready to go here. So can we the the the item please. Report from city clerk recommendation to receive and file the certification of the petition regarding the regulation of medical marijuana businesses and approve one of the following three alternative actions adopt the initiative ordinance without alteration to submit the initiative ordinance without alteration to the voters to be held on November 8th, 2016 or three. Adopt a report pursuant to California State Elections Code. Thank you. There's a motion and a second device. Marie Lowenthal. Thank you. And Mr. City Attorney, I'd like to add to the motion to prepare an analysis that comes back to this council. Mayor, Vice Mayor. I'm not sure which what the motion is as read by the city clerk. The council this evening has three options. It's it's items two and three from the three options to. To approve receipt of. The clerk certification and then to ask for a report. Yep. Okay. There's a motion in a second. So it's to approve the ballot, the ballot measure that the caucus sent over, but also concurrently to ask for a report that would come back to the council. Yes. Okay. There's a motion and a second. Ah. Any public comment on this? We'd love to take the vote expeditiously. Mr. Goodhue. If it's all possible, I would suggest. Taking whatever steps necessary and figuring out what the cost would be to oppose any actions. And that would end with that would result in having marijuana legalized within this city. If we're prepared to spend $90 million, urinate away $90 million on a city hall, we can certainly afford the millions to fight this off in court. To forestall what the police department has counseled against the consequences of having legalized marijuana here in this city if we think we had problems here tonight. The day would pale in comparison to what we'll have if marijuana is legalized in this city. Thank you. Any other public comment? We want to take a vote. Quickly, please. Just real quick. Although I would have preferred number one, I know there's no consensus or probably won't be. But I'm here just to say. That the. City clerk really did a fantastic job in what was a very tedious and detailed process. And I'm very proud of the professionalism. That they displayed. Thank you. Okay. Thank you. There's a motion and a second. Please cast your votes. Motion carries. Okay. Thank you. And now we're going to item and I'm sorry, jumping around. Give me 1/2. I think it's 12.",
+        "QA_pairs": [
+            [
+                "What was the next item number to be discussed in the meeting?",
+                "Item number 16."
+            ],
+            [
+                "What were the three alternative actions regarding the regulation of medical marijuana businesses?",
+                "Adopt the initiative ordinance, submit the initiative to voters, or adopt a report."
+            ],
+            [
+                "When was the initiative ordinance to be held?",
+                "On November 8th, 2016."
+            ],
+            [
+                "Who seconded the motion?",
+                "Marie Lowenthal."
+            ],
+            [
+                "What did the city attorney add to the motion?",
+                "To prepare an analysis."
+            ],
+            [
+                "What were the two items the council was to approve?",
+                "Receipt of the clerk certification and a report."
+            ],
+            [
+                "What did the caucus send over?",
+                "The ballot measure."
+            ],
+            [
+                "What did Mr. Goodhue suggest?",
+                "Opposing any actions legalizing marijuana."
+            ],
+            [
+                "What did the last public comment commend?",
+                "The city clerk's professionalism."
+            ],
+            [
+                "What was the result of the vote?",
+                "The motion carries."
+            ]
+        ]
+    },
+    {
+        "idx": 9,
+        "original_prompt": "Okay. 49, please. Each report from Human Resources recommendation to adopt a resolution approving an exception to the 180 day waiting period for public agencies to hire Charles Tripp for a limited duration to work in the Energy Resources Department citywide. Public comment, please. Yes, we have Dave Shukla. Dave Shukla. Did she go on sale? I have a host of questions about her facility, but it's really one of the important. So this is not dancing or prefer. Can we pipe offshore wind perhaps from the far side of Catalina Island where that generation sources know what is the relative toxicity of the ash that we pay people to bury? What are the processes? That are only known to a handful of people that might just by chance have already been lost this year. Four. Continue to keep the facility open. And do we have any estimates of the total amount of black carbon that has been absorbed into the local environment? From that facility. Over the past. Couple of years that we haven't really been monitoring anything by a whole bunch of other things as much nationally, but even locally we've kind of a bit less. I'd love to talk to Mr. Tripp, frankly. Thank you for your time. Thank you. That concludes public comment. Roll call vote. Actually, I need a motion in a second, please, on this item. Motion pocket. General roll cover, please. I. Sorry. But who is the maker of the Motion Mayor District? I think it was Pierson in the house. Thank you. District one. I'm District two, i district three, i district four. All right. District five. District seven. By. District eight. I. District nine. Ocean carries.",
+        "QA_pairs": [
+            [
+                "What is the recommendation from Human Resources?",
+                "To hire Charles Tripp."
+            ],
+            [
+                "Where is Charles Tripp expected to work?",
+                "In the Energy Resources Department."
+            ],
+            [
+                "Who made a public comment?",
+                "Dave Shukla."
+            ],
+            [
+                "What does Dave Shukla want to know about?",
+                "The relative toxicity of ash."
+            ],
+            [
+                "What else does Dave Shukla question?",
+                "The total amount of absorbed black carbon."
+            ],
+            [
+                "Who does Dave Shukla want to talk to?",
+                "Mr. Tripp."
+            ],
+            [
+                "What was required after the public comment?",
+                "A motion and a second."
+            ],
+            [
+                "Who made the motion?",
+                "It was Pierson."
+            ],
+            [
+                "What was the result of the roll call vote?",
+                "The motion carries."
+            ],
+            [
+                "What is the exception being approved for Charles Tripp?",
+                "The 180 day waiting period."
+            ]
+        ]
+    }
+]

llmlingua/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The cc-by-nc-sa-4.0 License [see LICENSE for details]
+# flake8: noqa
+from .prompt_compressor import PromptCompressor

llmlingua/utils.py

@@ -0,0 +1,98 @@
+import torch
+from torch.utils.data import Dataset
+import random, os
+import numpy as np
+import torch
+import string 
+
+class TokenClfDataset(Dataset):
+    def __init__(
+        self,
+        texts,
+        max_len=512,
+        tokenizer=None,
+        model_name="bert-base-multilingual-cased",
+    ):
+        self.len = len(texts)
+        self.texts = texts
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+        self.model_name = model_name
+        if "bert-base-multilingual-cased" in model_name:
+            self.cls_token = "[CLS]"
+            self.sep_token = "[SEP]"
+            self.unk_token = "[UNK]"
+            self.pad_token = "[PAD]"
+            self.mask_token = "[MASK]"
+        elif "xlm-roberta-large" in model_name:
+            self.bos_token = "<s>"
+            self.eos_token = "</s>"
+            self.sep_token = "</s>"
+            self.cls_token = "<s>"
+            self.unk_token = "<unk>"
+            self.pad_token = "<pad>"
+            self.mask_token = "<mask>"
+        else:
+            raise NotImplementedError()
+
+    def __getitem__(self, index):
+        text = self.texts[index]
+        tokenized_text = self.tokenizer.tokenize(text)
+
+        tokenized_text = (
+            [self.cls_token] + tokenized_text + [self.sep_token]
+        )  # add special tokens
+
+        if len(tokenized_text) > self.max_len:
+            tokenized_text = tokenized_text[: self.max_len]
+        else:
+            tokenized_text = tokenized_text + [
+                self.pad_token for _ in range(self.max_len - len(tokenized_text))
+            ]
+
+        attn_mask = [1 if tok != self.pad_token else 0 for tok in tokenized_text]
+
+        ids = self.tokenizer.convert_tokens_to_ids(tokenized_text)
+
+        return {
+            "ids": torch.tensor(ids, dtype=torch.long),
+            "mask": torch.tensor(attn_mask, dtype=torch.long),
+        }
+
+    def __len__(self):
+        return self.len
+
+
+def seed_everything(seed: int):
+    random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def is_begin_of_new_word(token, model_name, force_tokens, token_map):
+    if "bert-base-multilingual-cased" in model_name:
+        if token.lstrip("##") in force_tokens or token.lstrip("##") in set(token_map.values()):
+            return True
+        return not token.startswith("##")
+    elif "xlm-roberta-large" in model_name:
+        if token in string.punctuation or token in force_tokens or token in set(token_map.values()):
+            return True
+        return token.startswith("▁")
+    else:
+        raise NotImplementedError()
+        
+def replace_added_token(token, token_map):
+    for ori_token, new_token in token_map.items():
+        token = token.replace(new_token, ori_token)
+    return token
+    
+def get_pure_token(token, model_name):
+    if "bert-base-multilingual-cased" in model_name:
+        return token.lstrip("##")
+    elif "xlm-roberta-large" in model_name:
+        return token.lstrip("▁")
+    else:
+        raise NotImplementedError()

llmlingua/version.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2023 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+_MAJOR = "0"
+_MINOR = "1"
+# On master and in a nightly release the patch should be one ahead of the last
+# released build.
+_PATCH = "6"
+# This is mainly for nightly builds which have the suffix ".dev$DATE". See
+# https://semver.org/#is-v123-a-semantic-version for the semantics.
+_SUFFIX = ""
+
+VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR)
+VERSION = "{0}.{1}.{2}{3}".format(_MAJOR, _MINOR, _PATCH, _SUFFIX)

setup.cfg

@@ -0,0 +1,28 @@
+[isort]
+default_section = FIRSTPARTY
+ensure_newline_before_comments = True
+force_grid_wrap = 0
+include_trailing_comma = True
+known_first_party = sdtools
+known_third_party =
+    imblearn
+    numpy
+    pandas
+    pytorch-tabnet
+    scipy
+    sklearn
+    torch
+    torchaudio
+    torchvision
+    torch_xla
+    tqdm
+    xgboost
+
+line_length = 119
+lines_after_imports = 2
+multi_line_output = 3
+use_parentheses = True
+
+[flake8]
+ignore = E203, E501, E741, W503, W605
+max-line-length = 119

setup.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2023 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from setuptools import find_packages, setup
+
+# PEP0440 compatible formatted version, see:
+# https://www.python.org/dev/peps/pep-0440/
+#
+# release markers:
+#   X.Y
+#   X.Y.Z   # For bugfix releases
+#
+# pre-release markers:
+#   X.YaN   # Alpha release
+#   X.YbN   # Beta release
+#   X.YrcN  # Release Candidate
+#   X.Y     # Final release
+
+# version.py defines the VERSION and VERSION_SHORT variables.
+# We use exec here so we don't import allennlp whilst setting up.
+VERSION = {}  # type: ignore
+with open("llmlingua/version.py", "r") as version_file:
+    exec(version_file.read(), VERSION)
+
+INSTALL_REQUIRES = [
+    "transformers>=4.26.0",
+    "accelerate",
+    "torch",
+    "tiktoken",
+    "nltk",
+    "numpy",
+]
+QUANLITY_REQUIRES = [
+    "black==21.4b0",
+    "flake8>=3.8.3",
+    "isort>=5.5.4",
+    "pre-commit",
+    "pytest",
+    "pytest-xdist",
+]
+DEV_REQUIRES = INSTALL_REQUIRES + QUANLITY_REQUIRES
+
+setup(
+    name="llmlingua",
+    version=VERSION["VERSION"],
+    author="The LLMLingua team",
+    author_email="hjiang@microsoft.com",
+    description="To speed up LLMs' inference and enhance LLM's perceive of key information, compress the prompt and KV-Cache, which achieves up to 20x compression with minimal performance loss.",
+    long_description=open("README.md", encoding="utf8").read(),
+    long_description_content_type="text/markdown",
+    keywords="Prompt Compression, LLMs, Inference Acceleration, Black-box LLMs, Efficient LLMs",
+    license="MIT License",
+    url="https://github.com/microsoft/LLMLingua",
+    classifiers=[
+        "Intended Audience :: Science/Research",
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+    package_dir={"": "."},
+    packages=find_packages("."),
+    extras_require={
+        "dev": DEV_REQUIRES,
+        "quality": QUANLITY_REQUIRES,
+    },
+    install_requires=INSTALL_REQUIRES,
+    include_package_data=True,
+    python_requires=">=3.8.0",
+    zip_safe=False,
+)