Commit 7ad42d55 authored by Iulia Damian's avatar Iulia Damian
Browse files

added DataStructure and Test*added converter from json to

kotlin*implemented endpoint
parent c15b420c
Pipeline #73482 passed with stages
in 2 minutes and 17 seconds
......@@ -3,7 +3,7 @@
"name": "htc_sync_manager_faq.docx",
"question" : {
"style": "normal",
"type": "bold",
"format": "bold",
"size": 12
}
}
\ No newline at end of file
......@@ -3,7 +3,7 @@
"name": "https://www.stvo.de/info/faq",
"question" : {
"type": "text",
"css_selector": "table.category > tbody td.list-title > a",
"css_selector": "table.category > tbody td.list-title > a"
},
"answer": {
"type": "href",
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<bytecodeTargetLevel target="13" />
<bytecodeTargetLevel target="11" />
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_STRING" value="--warning-mode all" />
......
<?xml version="1.0" encoding="UTF-8"?>
<module external.linked.project.id="ktor-server:main" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="de.h_da.fbi.smebt.intentfinder" external.system.module.version="1.0-SNAPSHOT" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
<excludeFolder url="file://$MODULE_DIR$/build" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<component name="CheckStyle-IDEA-Module">
<option name="configuration">
<map />
</option>
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_13" default="true" project-jdk-name="13" project-jdk-type="JavaSDK">
<component name="FrameworkDetectionExcludesConfiguration">
<file type="web" url="file://$PROJECT_DIR$" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="SuppressABINotification">
<option name="modulesWithSuppressedNotConfigured">
<set>
<option value="ktor-server" />
</set>
</option>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ktor-server.iml" filepath="$PROJECT_DIR$/.idea/ktor-server.iml" />
</modules>
</component>
</project>
\ No newline at end of file
......@@ -60,6 +60,7 @@ dependencies {
testImplementation 'com.nhaarman.mockitokotlin2:mockito-kotlin:2.1.0' //Mockito-Kotlin
testImplementation 'org.mockito:mockito-inline:2.21.0'
testImplementation 'org.hamcrest:hamcrest:2.2'
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
}
compileKotlin {
......
package de.h_da.fbi.smebt.intentfinder.server
import com.squareup.moshi.JsonDataException
import de.h_da.fbi.smebt.intentfinder.server.nlp.PythonBridge
import de.h_da.fbi.smebt.intentfinder.server.sources.MoshiReader
import io.ktor.application.*
import io.ktor.features.*
import io.ktor.http.*
import io.ktor.http.content.*
import io.ktor.request.*
import io.ktor.response.*
import io.ktor.routing.*
import io.ktor.serialization.*
import kotlinx.serialization.json.Json
import registerUploadRoutes
import java.io.File
import java.lang.RuntimeException
......@@ -36,8 +40,9 @@ fun Application.module() {
val response = PythonBridge().getSummary("test bridge")
call.respond(response)
}
get("/") {
call.respondText("IntentFinder is available")
post("/{name}") {
}
// Definition eines Endpunkts zum Hochladen einer DOCX-Datei
......@@ -52,6 +57,76 @@ fun Application.module() {
// Definition eines Endpunkts zur Definition einer FAQ-Webseite mit JSON-Konfiguration
post("/faqRessource/{chatbotId}/{jsonStructure}"){
try{
//parse file title
var title =call.parameters["jsonStructure"]
if (title != null) {
title=title.replace(".json", "", false)
}
//set file directory
val uploadDir = "tmp"
val multipart = call.receiveMultipart()
var file = File("")
multipart.forEachPart { part ->
when (part) {
is PartData.FormItem -> {
if (part.name == "title") {
title = part.value
}
}
is PartData.FileItem -> {
val ext = File(part.originalFileName).extension
//generate file title
file = File(uploadDir, "upload-${System.currentTimeMillis()}-${title.hashCode()}.$ext")
val fileBytes = part.streamProvider().readBytes()
File(uploadDir).mkdir()
file.writeBytes(fileBytes)
}
}
part.dispose()
}
//building json String from File
val stringList = file.readLines()
var json = ""
for(element in stringList){
json+=element
}
try{
//Generate DataObject from json string
val moshiReader = MoshiReader("url")
val jsonO = moshiReader.getObjectFromJson(json)
}
catch (exception: Exception){
when(exception) {
is JsonDataException -> {
val errorMessage = "The uploaded file doesn't match the predefined structure. The only valid structure is: \n" +
" {\n" +
" \"type\": \"url\",\n" +
" \"name\": \"https://your.url\",\n" +
" \"question\" : \n " +
" {\n" +
" \"type\": \"text\",\n" +
" \"css_selector\": \"table.category > tbody td.list-title > a\"\n" +
" },\n" +
" \"answer\": \n " +
" {\n" +
" \"type\": \"[text|href]\",\n" +
" \"href_css_selector\": \"table.category > tbody td.list-title > a\",\n" +
" \"css_selector\": \"div.item-page > div:nth-child(5)\"\n" +
" }\n" +
"} \n"+
"The following field is missing or couldn't be parsed: "
call.respondText(errorMessage+exception.message.toString())
}
else -> throw exception
}
}
}
catch (exception: Exception){
call.respondText("The uploaded File can not be parsed. Please choose another .json File")
println(exception.message.toString())
}
}
......@@ -81,7 +156,7 @@ fun Application.module() {
}
}
registerUploadRoutes()
// registerUploadRoutes()
}
class InternalServerErrorException : RuntimeException()
......
package de.h_da.fbi.smebt.intentfinder.server.sources
data class JSONDocxConfig(
val type:String,
val name: String,
val question:QuestionDocx
)
data class QuestionDocx(
val style:String,
val format: String,
val size:Integer
)
\ No newline at end of file
package de.h_da.fbi.smebt.intentfinder.server.sources
/**
* Beispielstruktur:
* {
"type": "url",
"name": "https://www.stvo.de/info/faq",
"question" :
{
"type": "text",
"css_selector": "table.category > tbody td.list-title > a",
},
"answer":
{
"type": "href",
"href_css_selector": "table.category > tbody td.list-title > a",
"css_selector": "div.item-page > div:nth-child(5)"
}
}
**/
data class JSONUrlConfig(
val type:String,
val name:String,
val question:Question,
val answer:Answer
)
data class Question(
val type:String,
val css_selector:String
)
data class Answer(
val type:String,
val href_css_selector: String,
val css_selector:String
)
\ No newline at end of file
package de.h_da.fbi.smebt.intentfinder.server.sources
import com.squareup.moshi.JsonAdapter
import com.squareup.moshi.Moshi
import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
class MoshiReader(var configType:String) {
private val moshi = Moshi.Builder()
.addLast(KotlinJsonAdapterFactory())
.build()
private fun setAdapter():JsonAdapter<out Any>{
//Differ between url and docx
if(configType == "url") {
return moshi.adapter(JSONUrlConfig::class.java)
}
else {
return moshi.adapter(JSONDocxConfig::class.java)
}
}
@Throws(Exception::class)
fun getObjectFromJson(json: String): Any? {
return setAdapter().fromJson(json)
}
}
package de.h_da.fbi.smebt.intentfinder.server.sources
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.Assertions.*
internal class MoshiReaderTest {
@Test
fun getObjectFromJson() {
/*Test URL configs*/
val jsonTesStringUrl = "{\n" +
" \"type\": \"url\",\n" +
" \"name\": \"https://www.stvo.de/info/faq\",\n" +
" \"question\" : {\n" +
" \"type\": \"text\",\n" +
" \"css_selector\": \"table.category > tbody td.list-title > a\"\n" +
" },\n" +
" \"answer\": {\n" +
" \"type\": \"href\",\n" +
" \"href_css_selector\": \"table.category > tbody td.list-title > a\",\n" +
" \"css_selector\": \"div.item-page > div:nth-child(5)\"\n" +
" }\n" +
"}"
val moshiReaderUrl = MoshiReader("url")
val jsonUrl = moshiReaderUrl.getObjectFromJson(jsonTesStringUrl)
var nameUrl = ""
/*Cast json0 to the correct object*/
if (jsonUrl != null) {
if(jsonUrl is JSONUrlConfig?)
nameUrl = jsonUrl.name
if(jsonUrl is JSONDocxConfig?)
nameUrl = jsonUrl.name
}
assertEquals("https://www.stvo.de/info/faq", nameUrl)
/*Test DOCX configs*/
val jsonTesStringDocx = "{\n" +
" \"type\": \"doc\",\n" +
" \"name\": \"htc_sync_manager_faq.docx\",\n" +
" \"question\" : {\n" +
" \"style\": \"normal\",\n" +
" \"format\": \"bold\",\n" +
" \"size\": 12\n" +
" }\n" +
"}"
val moshiReaderDocx = MoshiReader("doc")
val jsonDocx = moshiReaderDocx.getObjectFromJson(jsonTesStringDocx)
var nameDocx = ""
/*Cast json0 to the correct object*/
if (jsonDocx != null) {
if(jsonDocx is JSONUrlConfig?)
nameDocx = jsonDocx.name
if(jsonDocx is JSONDocxConfig?)
nameDocx = jsonDocx.name
}
/*Test Docx configs*/
assertEquals("htc_sync_manager_faq.docx", nameDocx)
}
}
\ No newline at end of file
......@@ -76,8 +76,6 @@ pandas==1.2.4
scipy==1.6.2
protobuf==3.16.0
summarizer~=0.0.7
en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.1/en_core_web_sm-2.3.1.tar.gz
sacremoses==0.0.43
sentencepiece==0.1.94
tokenizers==0.9.4
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment