Commit 8d9bb448 authored by Areum Kim's avatar Areum Kim
Browse files

[Issue, #36] add files for MongoDB

parent 6ab542a6
Pipeline #77281 failed with stages
in 2 minutes and 35 seconds
......@@ -65,7 +65,14 @@ dependencies {
testImplementation 'com.nhaarman.mockitokotlin2:mockito-kotlin:2.1.0' //Mockito-Kotlin
testImplementation 'org.mockito:mockito-inline:2.21.0'
testImplementation 'org.hamcrest:hamcrest:2.2'
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
// for MongoDB
implementation("org.litote.kmongo:kmongo:4.2.7")
//jackson for reading json
implementation 'com.fasterxml.jackson.module:jackson-module-kotlin:2.12.1'
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.9.5'
implementation 'com.google.code.gson:gson:2.8.5'
}
compileKotlin {
......
......@@ -12,18 +12,32 @@ import de.h_da.fbi.smebt.intentfinder.server.routing.summarization.summarization
import de.h_da.fbi.smebt.intentfinder.server.routing.tags.Tags
import de.h_da.fbi.smebt.intentfinder.server.routing.upload.uploadRoutes
import de.h_da.fbi.smebt.intentfinder.server.upload.registerApiRoutes
import com.squareup.moshi.JsonDataException
import de.h_da.fbi.smebt.intentfinder.server.nlp.PythonBridge
import de.h_da.fbi.smebt.intentfinder.server.nlp.dto.Summary
import de.h_da.fbi.smebt.intentfinder.server.sources.MongoData
import de.h_da.fbi.smebt.intentfinder.server.sources.MoshiReader
import io.ktor.application.*
import io.ktor.features.*
import io.ktor.gson.*
import io.ktor.http.*
import io.ktor.http.content.*
import io.ktor.request.*
import io.ktor.response.*
import io.ktor.routing.*
import io.ktor.serialization.*
import kotlinx.serialization.json.Json
import kotlin.reflect.KType
import java.io.File
import java.lang.RuntimeException
fun main(args: Array<String>): Unit = io.ktor.server.netty.EngineMain.main(args)
fun Application.module() {
val data = MongoData()
install(OpenAPIGen) {
info {
version = "0.0.1"
......@@ -48,7 +62,13 @@ fun Application.module() {
}
install(ContentNegotiation) {
json()
gson {
setPrettyPrinting()
}
json(Json {
prettyPrint = true
})
}
install(StatusPages) {
......@@ -69,6 +89,134 @@ fun Application.module() {
get("/openapi.json") {
call.respond(GsonBuilder().create().toJson(application.openAPIGen.api.serialize()))
}
get("/summary") {
val response = PythonBridge().getSummary("test bridge",10)
call.respond(response)
}
get("/") {
call.respondText("IntentFinder is available")
}
// Definition eines Endpunkts zum Hochladen einer DOCX-Datei
post("/file/{chatbotId}") {
call.respondText("file was successful uploaded")
}
// Definition endpoint zur Änderung eine bereits existierende docx
put("/file/{chatbotId}/{id}/{filename}"){
}
// Definition eines Endpunkts zur Definition einer FAQ-Webseite mit JSON-Konfiguration
post("/faqRessource/{chatbotId}/{jsonStructure}"){
try{
//parse file title
var title =call.parameters["jsonStructure"]
if (title != null) {
title= title!!.replace(".json", "", false)
}
//set file directory
val uploadDir = "tmp"
val multipart = call.receiveMultipart()
var file = File("")
multipart.forEachPart { part ->
when (part) {
is PartData.FormItem -> {
if (part.name == "title") {
title = part.value
}
}
is PartData.FileItem -> {
val ext = File(part.originalFileName).extension
//generate file title
file = File(uploadDir, "upload-${System.currentTimeMillis()}-${title.hashCode()}.$ext")
val fileBytes = part.streamProvider().readBytes()
File(uploadDir).mkdir()
file.writeBytes(fileBytes)
}
}
part.dispose()
}
//building json String from File
val stringList = file.readLines()
var json = ""
for(element in stringList){
json+=element
}
try{
//Generate DataObject from json string
val moshiReader = MoshiReader("url")
val jsonO = moshiReader.getObjectFromJson(json)
}
catch (exception: Exception){
when(exception) {
is JsonDataException -> {
val errorMessage = "The uploaded file doesn't match the predefined structure. The only valid structure is: \n" +
" {\n" +
" \"type\": \"url\",\n" +
" \"name\": \"https://your.url\",\n" +
" \"question\" : \n " +
" {\n" +
" \"type\": \"text\",\n" +
" \"css_selector\": \"table.category > tbody td.list-title > a\"\n" +
" },\n" +
" \"answer\": \n " +
" {\n" +
" \"type\": \"[text|href]\",\n" +
" \"href_css_selector\": \"table.category > tbody td.list-title > a\",\n" +
" \"css_selector\": \"div.item-page > div:nth-child(5)\"\n" +
" }\n" +
"} \n"+
"The following field is missing or couldn't be parsed: "
call.respondText(errorMessage+exception.message.toString())
}
else -> throw exception
}
}
}
catch (exception: Exception){
call.respondText("The uploaded File can not be parsed. Please choose another .json File")
println(exception.message.toString())
}
}
// get faq with Json Configuration
get("/faqRessource"){
//Rückgabe Json Object (vgl. #38)
}
// Definition eines Endpunkts zur Definition einer Docx-Datei mit JSON-Konfiguration
post("/docxRessource/{chatbotId}/{filename}"){
}
// get docx file with Json Configuration
get("/docxRessource"){
//Rückgabe Json Object (vgl. #37)
}
get("/files"){
var allFiles: MutableList<AllFiles> = mutableListOf()
data.getAllFaqData().forEach { faqData ->
if(faqData.type == "docx"){
allFiles.add(AllFiles(faqData.name, faqData.id.toString(), faqData.status))
}
}
call.respond(allFiles)
}
// Routen ohne Funktionalität
routing{
}
}
apiRouting {
......@@ -89,4 +237,5 @@ fun Application.module() {
class InternalServerErrorException : RuntimeException()
data class AllFiles(var filename:String, var id: String, var status: String)
package de.h_da.fbi.smebt.intentfinder.server.sources
import de.h_da.fbi.smebt.intentfinder.server.nlp.dto.Summary
import de.h_da.fbi.smebt.intentfinder.server.nlp.dto.SummaryBody
import org.bson.codecs.pojo.annotations.BsonId
import org.litote.kmongo.Id
import org.litote.kmongo.newId
import java.text.DateFormat
import java.text.SimpleDateFormat
import java.util.TimeZone
import java.util.Date
data class FAQData(
@BsonId val id: Id<FAQData>,
var type: String,
var name: String,
var creationTimestamp: String,
var changeTimestamp: MutableList<String>,
var status: String,
var dialogs: MutableList<Dialog>,
var unprocessed: MutableList<String>
)
{
var jsonDocxConfig: JSONDocxConfig ?= null
var jsonUrlConfig : JSONUrlConfig ?= null
constructor(): this(newId(), "docx", "name", setTimestamp(),
mutableListOf("2021-06-13T11:45", "2021-06-13T11:46"),
"status", mutableListOf(Dialog()), mutableListOf("text 1", "text 2"))
constructor(id: Id<FAQData>, type: String, name: String,
creationTimestamp: String, changeTimestamp: MutableList<String>
, status: String, dialogs: MutableList<Dialog>, unprocessed: MutableList<String>,
jsonDocxConfig: JSONDocxConfig): this(id, type, name, creationTimestamp, changeTimestamp, status, dialogs,
unprocessed)
{
this.jsonDocxConfig = jsonDocxConfig
}
constructor(id: Id<FAQData>, type: String, name: String,
creationTimestamp: String, changeTimestamp: MutableList<String>,
status: String, dialogs: MutableList<Dialog>,
unprocessed: MutableList<String>, jsonUrlConfig : JSONUrlConfig): this(id, type, name, creationTimestamp,
changeTimestamp, status, dialogs,
unprocessed)
{
this.jsonUrlConfig = jsonUrlConfig
}
}
data class Dialog (
val id: String,
val question: String,
val alternativeQuestions: MutableList<String>,
val answer: String,
val summaries: MutableList<Summary> )
{
constructor(): this("d1", "What time is it?", mutableListOf("time?", "what is the time?"), "12h00",
mutableListOf(Summary(200, "message", SummaryBody("strategy",0.0,"summary"))))
}
// set current time
fun setTimestamp(): String {
val tz: TimeZone = TimeZone.getTimeZone("UTC")
val df: DateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'") // Quoted "Z" to indicate UTC, no timezone offset
df.timeZone = tz
return df.format(Date())
}
\ No newline at end of file
......@@ -5,8 +5,12 @@ data class JSONDocxConfig(
var name: String,
val question:QuestionDocx
)
{
constructor(): this("none", "none", QuestionDocx("none", "none", 0))
}
data class QuestionDocx(
val style:String,
val format: String,
val size:Integer
val size:Int
)
\ No newline at end of file
......@@ -22,9 +22,11 @@ data class JSONUrlConfig(
val name:String,
val question:Question,
val answer:Answer
)
{
constructor(): this("none", "none",
Question("none", "none"), Answer("none", "none", "none"))
}
data class Question(
val type:String,
......@@ -33,5 +35,6 @@ data class Question(
)
data class Answer(
val type:String,
val href_css_selector: String,
val css_selector:String
)
\ No newline at end of file
package de.h_da.fbi.smebt.intentfinder.server.sources
import com.mongodb.client.MongoCollection
import com.mongodb.client.MongoDatabase
import com.mongodb.client.model.Filters.eq
import de.h_da.fbi.smebt.intentfinder.server.nlp.dto.Summary
import org.bson.Document
import org.bson.types.ObjectId
import org.litote.kmongo.*
class MongoData {
var col: MongoCollection<FAQData>
private val database: MongoDatabase
private val dbName = "chats"
private val collectionName = "faqData"
init {
val client = KMongo.createClient()
database = client.getDatabase(dbName)
col = database.getCollection<FAQData>(collectionName)
}
fun deleteAllFaqData() {
col.deleteMany(Document())
}
fun insertFaqDataDocxInMongo(faqData: FAQData): Id<FAQData> {
if(faqData.type == "docx")
col.insertOne(
FAQData(
faqData.id,
faqData.type,
faqData.name,
faqData.creationTimestamp,
faqData.changeTimestamp,
faqData.status,
faqData.dialogs,
faqData.unprocessed,
faqData.jsonDocxConfig!!
)
)
return faqData.id //id aus der Mongo
}
fun insertFaqDataUrlInMongo(faqData: FAQData): Id<FAQData> {
if(faqData.type == "url")
col.insertOne(
FAQData(
faqData.id,
faqData.type,
faqData.name,
faqData.creationTimestamp,
faqData.changeTimestamp,
faqData.status,
faqData.dialogs,
faqData.unprocessed,
faqData.jsonUrlConfig!!
)
)
return faqData.id //id aus der Mongo
}
fun getFaqDataById(id: String): FAQData? {
return col.find(eq("_id", ObjectId(id))).first()
}
fun getAllFaqData(): List<FAQData> {
val result = col.find()
return result.toList()
}
fun addTimestamp(id: String): Boolean {
var status = true
if (getFaqDataById(id).toString() == "null") {
status = false
} else { // add time for every update
val changeTimeStamp = getFaqDataById(id)?.changeTimestamp
changeTimeStamp?.add(setTimestamp())
updateChangeTimeStamp(id, changeTimeStamp!!)
}
return status
}
fun updateFaqDataName(id: String, name: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("name", name)))
return addTimestamp(id)
}
fun updateFaqDataType(id: String, type: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("type", type)))
return addTimestamp(id)
}
fun updateFaqDataJsonDocxConfigType(id: String, type: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonDocxConfig.type", type)))
return addTimestamp(id)
}
fun updateFaqDataJsonDocxConfigName(id: String, name: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonDocxConfig.name", name)))
return addTimestamp(id)
}
fun updateFaqDataJsonDocxConfigQuestionStyle(id: String, style: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonDocxConfig.question.style", style)))
return addTimestamp(id)
}
fun updateFaqDataJsonDocxConfigQuestionFormat(id: String, format: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonDocxConfig.question.format", format)))
return addTimestamp(id)
}
fun updateFaqDataJsonDocxConfigQuestionSize(id: String, size: Int): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonDocxConfig.question.size", size)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigType(id:String, urlType: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.type", urlType)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigName(id:String, urlName: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.name", urlName)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigQuestionType(id:String, questionType: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.question.type", questionType)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigQuestionCss(id:String, css_selector: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.question.css_selector", css_selector)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigAnswerType(id:String, answerType: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.answer.type", answerType)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigAnswerHref(id:String, AnswerHref: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.answer.href_css_selector", AnswerHref)))
return addTimestamp(id)
}
fun updateFaqDataJsonUrlConfigAnswerCss(id:String, AnswerCss: String): Boolean{
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("jsonUrlConfig.answer.css_selector", AnswerCss)))
return addTimestamp(id)
}
fun updateFaqDataStatus(id: String, status: String): Boolean {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("status", status)))
return addTimestamp(id)
}
fun updateChangeTimeStamp(id: String, timeStamp: List<String>) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("changeTimestamp", timeStamp)))
}
fun updateFaqDataDialogQuestion(id: String, question: String, dialogId: String): Boolean {
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed{ dialogIndex, data ->
if (data.id == dialogId) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("dialogs.$dialogIndex.question", question)))
return addTimestamp(id)
}
}
return false
}
fun updateFaqDataDialogAnswer(id: String, answer: String, dialogId: String): Boolean {
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed{ dialogIndex, data ->
if (data.id == dialogId) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set", Document("dialogs.$dialogIndex.answer", answer)))
return addTimestamp(id)
}
}
return false
}
fun updateFaqDataDialogAlternativeQuestion(id: String, alternativeQuestionIndex: Int, newAlternativeQuestion: String,
dialogId: String): Boolean {
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed { dialogIndex, dialog ->
if (dialog.id == dialogId) {
if(faqDialogData.size > alternativeQuestionIndex) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set",
Document("dialogs.$dialogIndex.alternativeQuestions.$alternativeQuestionIndex", newAlternativeQuestion)))
return addTimestamp(id)
}
}
}
return false
}
fun updateFaqDataDialogSummary(id: String, summaryIndex: Int, summary: Summary, dialogId: String): Boolean{
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed { dialogIndex, dialog ->
if (dialog.id == dialogId) {
if(faqDialogData.size > summaryIndex) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set",
Document("dialogs.$dialogIndex.summaries.$summaryIndex", summary)
)
)
return addTimestamp(id)
}
}
}
return false
}
fun updateFaqDataDialogSummaryStrategy(id: String, summaryIndex: Int, newSummaryStrategy: String,
dialogId: String): Boolean {
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed { dialogIndex, dialog ->
if (dialog.id == dialogId) {
if(faqDialogData.size > summaryIndex) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set",
Document("dialogs.$dialogIndex.summaries.$summaryIndex.strategy", newSummaryStrategy)))
return addTimestamp(id)
}
}
}
return false
}
fun updateFaqDataDialogSummaryQuality(id: String, summaryIndex: Int, newSummaryQuality: Double,
dialogId: String): Boolean {
val faqDialogData = getFaqDataById(id)?.dialogs
faqDialogData?.forEachIndexed{ dialogIndex, dialog ->
if (dialog.id == dialogId) {
if(faqDialogData.size > summaryIndex) {
col.updateOne(eq("_id", ObjectId(id)), Document("\$set",
Document("dialogs.$dialogIndex.summaries.$summaryIndex.quality", newSummaryQuality)
)
)