Skip to content

Commit

Permalink
added corrections to the feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
@justinhunt
justinhunt committed Oct 31, 2024
1 parent 446a1a8 commit 347e448
Show file tree
Hide file tree
Showing 7 changed files with 65 additions and 59 deletions.
40 changes: 20 additions & 20 deletions classes/constants.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,16 @@

class constants {

const RELEVANCE_NONE=0;
const RELEVANCE_QTEXT=1;
const RELEVANCE_COMPARISON=2;
const LANGUAGES = ['ar-ae', 'ar-sa', 'eu-es', 'bg-bg', 'hr-hr', 'zh-cn', 'cs-cz', 'da-dk', 'nl-nl', 'nl-be', 'en-us', 'en-gb',
const RELEVANCE_NONE = 0;
const RELEVANCE_QTEXT = 1;
const RELEVANCE_COMPARISON = 2;
const LANGUAGES = ['ar-ae', 'ar-sa', 'eu-es', 'bg-bg', 'hr-hr', 'zh-cn', 'cs-cz', 'da-dk', 'nl-nl', 'nl-be', 'en-us', 'en-gb',
'en-au', 'en-nz', 'en-za', 'en-in', 'en-ie', 'en-wl', 'en-ab', 'fa-ir', 'fil-ph', 'fi-fi', 'fr-ca', 'fr-fr', 'de-de',
'de-at', 'de-ch', 'hi-in', 'el-gr', 'he-il', 'hu-hu', 'id-id', 'is-is', 'it-it', 'ja-jp', 'ko-kr', 'lt-lt', 'lv-lv',
'mi-nz', 'ms-my', 'mk-mk', 'no-no', 'pl-pl', 'pt-br', 'pt-pt', 'ro-ro', 'ru-ru', 'es-us', 'es-es', 'sk-sk', 'sl-si',
'sr-rs', 'sv-se', 'ta-in', 'te-in', 'tr-tr', 'uk-ua', 'vi-vn'];

const RESPONSE_FORMATS = ['plain','editor','monospaced','audio'];
const RESPONSE_FORMATS = ['plain', 'editor', 'monospaced', 'audio'];
const EXTRA_FIELDS = ['responseformat',
'responsefieldlines',
'minwordlimit',
Expand All @@ -54,7 +54,7 @@ class constants {
'responselanguage',
'feedbacklanguage',
'relevance',
'relevanceanswer'
'relevanceanswer',
];

/**
Expand All @@ -64,12 +64,12 @@ class constants {
* @return array
*/
public static function get_languages($includeauto=false) {
$responselanguages=[];
$responselanguages = [];
foreach (self::LANGUAGES as $langcode) {
$responselanguages[$langcode] = get_string($langcode,"qtype_aitext");
$responselanguages[$langcode] = get_string($langcode, "qtype_aitext");
}
if($includeauto){
$responselanguages['currentlanguage'] = get_string('currentlanguage',"qtype_aitext");
$responselanguages['currentlanguage'] = get_string('currentlanguage', "qtype_aitext");
}
return $responselanguages;
}
Expand All @@ -81,9 +81,9 @@ public static function get_languages($includeauto=false) {
* @return array
*/
public static function get_response_formats() {
$responseformats=[];
$responseformats = [];
foreach (self::RESPONSE_FORMATS as $theformat) {
$responseformats[$theformat] = get_string('format' . $theformat,"qtype_aitext");
$responseformats[$theformat] = get_string('format' . $theformat, "qtype_aitext");
}
return $responseformats;
}
Expand All @@ -95,27 +95,27 @@ public static function get_response_formats() {
* @return array
*/
public static function get_time_limits() {
$opts = array(
$opts = [
0 => get_string("notimelimit", "qtype_aitext"),
30 => get_string("xsecs", "qtype_aitext", '30'),
45 => get_string("xsecs", "qtype_aitext", '45'),
60 => get_string("onemin", "qtype_aitext"),
90 => get_string("oneminxsecs", "qtype_aitext", '30'),
);
for($x=2;$x<=30;$x++){
$opts[$x*60]=get_string("xmins", "qtype_aitext", $x);
$opts[($x*60)+30]=get_string("xminsecs", "qtype_aitext", array('minutes' => $x, 'seconds' => 30));
];
for ($x = 2; $x <= 30; $x++) {
$opts[$x * 60] = get_string("xmins", "qtype_aitext", $x);
$opts[($x * 60) + 30] = get_string("xminsecs", "qtype_aitext", ['minutes' => $x, 'seconds' => 30]);
}
return $opts;
}

public static function get_relevance_opts() {
$opts = array(
$opts = [
self::RELEVANCE_NONE => get_string("relevance_none", "qtype_aitext"),
self::RELEVANCE_QTEXT => get_string("relevancetoqtext", "qtype_aitext"),
self::RELEVANCE_COMPARISON => get_string("relevancetocomparison", "qtype_aitext")
);
self::RELEVANCE_COMPARISON => get_string("relevancetocomparison", "qtype_aitext"),
];
return $opts;
}

}
}
2 changes: 1 addition & 1 deletion classes/external.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public static function fetch_ai_grade_parameters(): external_function_parameters
* @param integer $defaultmark
* @param string $prompt
* @param string $marksscheme
* @return void
* @return array
*/
public static function fetch_ai_grade($response, $defaultmark, $prompt, $marksscheme) {
// Get our AI helper.
Expand Down
4 changes: 2 additions & 2 deletions db/upgrade.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ function xmldb_qtype_aitext_upgrade($oldversion) {
$currentjsonprompt = preg_replace('/\s+/', ' ', trim($currentjsonprompt));

if ($currentjsonprompt == $originaljsonprompt) {
$newprompt = "Return only a JSON object which enumerates a set of 3 elements.";
$newprompt .= ' The JSON object should be in this format: {"feedback":"string","marks":"number", "relevance": "number"}';
$newprompt = "Return only a JSON object which enumerates a set of 4 elements.";
$newprompt .= ' The JSON object should be in this format: {"feedback":"string","correctedtext":"string",marks":"number", "relevance": "number"}';
$newprompt .= " where marks is a single number summing all marks.";
set_config('jsonprompt', $newprompt, 'qtype_aitext');
}
Expand Down
3 changes: 2 additions & 1 deletion lang/en/qtype_aitext.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,4 +191,5 @@
$string['retry'] = 'Retry';
$string['currentwordcount'] = 'Total Words';
$string['submissionrelevance'] = 'Relevance: {$a}%';
$string['relevanceheader'] = 'Relevance';
$string['relevanceheader'] = 'Relevance';
$string['correctedtext'] = 'Corrections:';
56 changes: 31 additions & 25 deletions question.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class qtype_aitext_question extends question_graded_automatically_with_countback

/**
* LLM Model, will vary between AI systems, e.g. gpt4 or llama3
* @var mixed $model Store the llm model used for the question.
*/
public $model;
Expand Down Expand Up @@ -196,8 +196,8 @@ public function grade_response(array $response): array {
// Calculate the fraction of the marks but AI sometimes gives more marks than possible, so cap it.
$fraction = $contentobject->marks > $this->defaultmark ? 1 : $contentobject->marks / $this->defaultmark;

//relevance penalty
if(isset($contentobject->relevance) && $contentobject->relevance!==null){
// Relevance penalty.
if (isset($contentobject->relevance) && $contentobject->relevance !== null) {
$fraction = $fraction * ($contentobject->relevance * 0.01);
}

Expand All @@ -216,7 +216,7 @@ public function grade_response(array $response): array {
* @param string $markscheme The mark scheme.
* @package qtype_aitext
*/
private function queue_ai_processing(string $answer, string $aiprompt, float $defaultmark, string $markscheme): void{
private function queue_ai_processing(string $answer, string $aiprompt, float $defaultmark, string $markscheme): void {
global $DB;
$data = [
'activity' => 'qtype_aitext',
Expand Down Expand Up @@ -262,40 +262,40 @@ public function insert_feedback_and_prompt($fullaiprompt, $contentobject): void
* @return string;
*/
public function build_full_ai_prompt($response, $aiprompt, $defaultmark, $markscheme): string {
$prompttemplate ="You are evaluating a student's {{responselanguage}} language response to a question. ";
$prompttemplate .= " {{jsonprompt}}. ";
//$prompttemplate .= " Return only a JSON object which enumerates a set of 3 elements.The JSON object should be in this format: {"feedback":"string","marks":"number", "relevance": "number"} where marks is a single number summing all marks. ";
$prompttemplate = "You are evaluating a student's {{responselanguage}} language response to a question. ";
$prompttemplate .= " {{jsonprompt}}. ";
// $prompttemplate .= " Return only a JSON object which enumerates a set of 3 elements.The JSON object should be in this format: {"feedback":"string","marks":"number", "relevance": "number"} where marks is a single number summing all marks. ";

$prompttemplate .= get_config('qtype_aitext', 'prompt');
//$prompttemplate = "In [{{responsetext}}] analyse but do not mention the part between [[ and ]] as follows: ";
// $prompttemplate = "In [{{responsetext}}] analyse but do not mention the part between [[ and ]] as follows: ";

$prompttemplate .= " {{{aiprompt}}}";
//$prompttemplate .= " Explain if there is anything wrong with the grammar and spelling in the text";
$prompttemplate .= " {{{aiprompt}}}";
// $prompttemplate .= " Explain if there is anything wrong with the grammar and spelling in the text";

if(!empty($markscheme)){
$prompttemplate .= " Set marks in the json object according to the following criteria: {The maximum score is {{maximumscore}}. {{markscheme}}}";
//$prompttemplate .=" Set marks in the json object according to the following criteria: {The total score is 5. Deduct a point from the maximum score for each grammar or spelling mistake.}"
}else{
$prompttemplate .= " Set marks to null in the json object.";
if (!empty($markscheme)) {
$prompttemplate .= " Set marks in the json object according to the following criteria: {The maximum score is {{maximumscore}}. {{markscheme}}}";
// $prompttemplate .=" Set marks in the json object according to the following criteria: {The total score is 5. Deduct a point from the maximum score for each grammar or spelling mistake.}"
} else {
$prompttemplate .= " Set marks to null in the json object.";
}
switch($this->relevance){
case constants::RELEVANCE_QTEXT:
$prompttemplate .= " Calculate the relevance of the answer (percentage) to the following question : {{{questiontext}}}";
$prompttemplate .= " Calculate the relevance of the answer (percentage) to the following question : {{{questiontext}}}";
break;
case constants::RELEVANCE_COMPARISON:
$prompttemplate .= " Calculate the relevance of the answer (percentage) to the extent it contains similar concepts to the following model answer : {{{relevanceanswer}}}";
$prompttemplate .= " Calculate the relevance of the answer (percentage) to the extent it contains similar concepts to the following model answer : {{{relevanceanswer}}}";
break;
case constants::RELEVANCE_NONE:
default:
$prompttemplate .= " Set relevance to null in the json object.";
$prompttemplate .= " Set relevance to null in the json object.";
break;
}
$prompttemplate .= " Translate the feedback to the language: {{feedbacklanguage}}.";
$prompttemplate .= " Translate the feedback to the language: {{feedbacklanguage}}.";

//set up the parameters to merge with the prompt template
// set up the parameters to merge with the prompt template
$responselanguage = empty($this->responselanguage) ? 'en-us' : $this->responselanguage;
$responselanguagename = get_string($responselanguage, 'qtype_aitext');
$params= [
$params = [
'[responsetext]' => '[[' . strip_tags($response) . ']]',
'{{aiprompt}}' => trim($aiprompt),
'{{maximumscore}}' => $defaultmark,
Expand All @@ -305,7 +305,7 @@ public function build_full_ai_prompt($response, $aiprompt, $defaultmark, $marksc
'{{questiontext}}' => strip_tags($this->questiontext),
'{{feedbacklanguage}}' => $this->feedbacklanguage == 'currentlanguage' || empty($this->feedbacklanguage) ?
current_language() : $this->feedbacklanguage,
'{{responselanguage}}' => $responselanguagename
'{{responselanguage}}' => $responselanguagename,
];
$prompt = strtr($prompttemplate, $params);
return $prompt;
Expand All @@ -328,12 +328,18 @@ public function process_feedback(string $feedback) {
$contentobject->feedback = trim($contentobject->feedback);
$contentobject->feedback = preg_replace(['/\[\[/', '/\]\]/'], '"',
$contentobject->feedback);
if(isset($contentobject->relevance) && $contentobject->relevance!==null){
// Relevance.
if (isset($contentobject->relevance) && $contentobject->relevance !== null) {
$contentobject->feedback .= ' ' . get_string('submissionrelevance', 'qtype_aitext', $contentobject->relevance);
}
// Corrections.
if (isset($contentobject->correctedtext) && !empty($contentobject->correctedtext)) {
$contentobject->feedback .= '<br/>' . get_string('correctedtext', 'qtype_aitext') . '<br/>';
$contentobject->feedback .= $contentobject->correctedtext;
}
$disclaimer = get_config('qtype_aitext', 'disclaimer');
$disclaimer = str_replace("[[model]]", $this->model, $disclaimer);
$contentobject->feedback .= ' '.$this->llm_translate($disclaimer);
$contentobject->feedback .= '<br/>' . $this->llm_translate($disclaimer);
} else {
$contentobject = (object) [
"feedback" => $feedback,
Expand Down Expand Up @@ -414,7 +420,7 @@ public function summarise_response(array $response) {
if (isset($response['answer']) && isset($response['answerformat'])) {
$output = question_utils::to_plain_text($response['answer'],
$response['answerformat'], ['para' => false]);
}else if(isset($response['answer'])){
} else if (isset($response['answer'])) {
$output = question_utils::to_plain_text($response['answer'],
FORMAT_HTML, ['para' => false]);
}
Expand Down
2 changes: 1 addition & 1 deletion questiontype.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public function save_question_options($formdata) {
$options->responsetemplate = $formdata->responsetemplate['text'];
$options->responsetemplateformat = $formdata->responsetemplate['format'];

//audior recording related options
// Audio recording related options.
$options->responselanguage = $formdata->responselanguage;
$options->feedbacklanguage = $formdata->feedbacklanguage;
$options->maxtime = $formdata->maxtime;
Expand Down
17 changes: 8 additions & 9 deletions renderer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ protected function class_name() {
}
/**
* Return a read only version of the response areay. Typically for after
* a quesiton has been answered and the response cannot be modified.
* a question has been answered and the response cannot be modified.
* @param string $name
* @param question_attempt $qa
* @param question_attempt_step $step
Expand Down Expand Up @@ -528,10 +528,10 @@ public function response_area_input($name, $qa, $step, $lines, $context) {
$inputname = $qa->get_qt_field_name($name);
$id = $inputname . '_id';

//get existing response and its wordcount
// get existing response and its wordcount
list($draftitemid, $response) = $this->prepare_response_for_editing($name, $step, $context);
if(!empty($response)) {
$wordcount = count_words($response); //fetch this from existing response
$wordcount = count_words($response); // fetch this from existing response
}else{
$wordcount = 0;
}
Expand All @@ -540,7 +540,6 @@ public function response_area_input($name, $qa, $step, $lines, $context) {
// Var inputname - is the name of the input field (hidden in this case).
// Var id - is the id of the input field (hidden in this case).


$responselabel = $this->displayoptions->add_question_identifier_to_label(get_string('answeraudio', 'qtype_aitext'));
$output = html_writer::tag('label', $responselabel, [
'class' => 'sr-only',
Expand All @@ -549,21 +548,21 @@ public function response_area_input($name, $qa, $step, $lines, $context) {
$output .= html_writer::start_tag('div', ['class' =>
$this->class_name() . ' qtype_aitext_response']);

//add the audio recorder
// add the audio recorder
$responselanguage = $question->responselanguage;
$output .= $this->render_from_template('qtype_aitext/audiorecorder', [
'id' => $id,
'inputname' => $inputname,
'safeid' => str_replace(':', '_', $id),
'haveresponse' => empty($response) ? false : true,
'haveresponse' => empty($response) ? false : true,
'response' => $response,
'waveheight' => 75,
'asrurl' => 'https://useast.ls.poodll.com/transcribe',//TO DO - get the selected region from the question settings
'region' => 'useast1', //TO DO - wire this up with settings from the question
'asrurl' => 'https://useast.ls.poodll.com/transcribe', // TO DO - get the selected region from the question settings
'region' => 'useast1', // TO DO - wire this up with settings from the question
'language' => $responselanguage,
'maxtime' => $question->maxtime,
'wordcount' => $wordcount,
'cancountwords'=> !in_array($responselanguage,['ja-jp','zh-cn','zh-tw','ko-kr']) ,
'cancountwords' => !in_array($responselanguage, ['ja-jp', 'zh-cn', 'zh-tw', 'ko-kr']) ,
]);

$output .= html_writer::end_tag('div');
Expand Down

0 comments on commit 347e448

Please sign in to comment.