機械学習の組み込み

type Query {
  recognizeTextFromImage: String @predictions(actions: [identifyText])
}

query RecognizeTextFromImage($input: RecognizeTextFromImageInput!) {
  recognizeTextFromImage(input: { identifyText: { key: "myimage.jpg" } })
}

type Query {
  recognizeLabelsFromImage: [String] @predictions(actions: [identifyLabels])
}

query RecognizeLabelsFromImage($input: RecognizeLabelsFromImageInput!) {
  recognizeLabelsFromImage(input: { identifyLabels: { key: "myimage.jpg" } })
}

type Query {
  translate: String @predictions(actions: [translateText])
}

query TranslateText($input: TranslateTextInput!) {
  translate(
    input: {
      translateText: {
        sourceLanguage: "en"
        targetLanguage: "de"
        text: "Translate me"
      }
    }
  )
}

type Query {
  textToSpeech: String @predictions(actions: [convertTextToSpeech])
}

query ConvertTextToSpeech($input: ConvertTextToSpeechInput!) {
  textToSpeech(
    input: {
      convertTextToSpeech: {
        voiceID: "Nicole"
        text: "Hello from AWS Amplify!"
      }
    }
  )
}

type Query {
  speakTranslatedImageText: String
    @predictions(actions: [identifyText, translateText, convertTextToSpeech])
}

query SpeakTranslatedImageText($input: SpeakTranslatedImageTextInput!) {
  speakTranslatedImageText(
    input: {
      identifyText: { key: "myimage.jpg" }
      translateText: { sourceLanguage: "en", targetLanguage: "es" }
      convertTextToSpeech: { voiceID: "Conchita" }
    }
  )
}

import React, { useState } from 'react';
import { Amplify } from 'aws-amplify';
import { uploadData, getUrl } from 'aws-amplify/storage';
import { generateClient } from 'aws-amplify/api';
import config from './amplifyconfiguration.json';
import { speakTranslatedImageText } from './graphql/queries';

/* Configure Exports */
Amplify.configure(config);

const client = generateClient();

function SpeakTranslatedImage() {
  const [src, setSrc] = useState('');
  const [img, setImg] = useState('');

  function putS3Image(event) {
    const file = event.target.files[0];
    uploadData({
      key: file.name,
      data: file
    })
      .result.then(async (result) => {
        setSrc(await speakTranslatedImageTextOP(result.key));
        setImg((await getUrl({ key: result.key })).url.toString());
      })
      .catch((err) => console.log(err));
  }

  return (
    <div className="Text">
      <div>
        <h3>Upload Image</h3>
        <input
          type="file"
          accept="image/jpeg"
          onChange={(event) => {
            putS3Image(event);
          }}
        />
        <br />
        {img && <img src={img}></img>}
        {src && (
          <div>
            <audio id="audioPlayback" controls>
              <source id="audioSource" type="audio/mp3" src={src} />
            </audio>
          </div>
        )}
      </div>
    </div>
  );
}

async function speakTranslatedImageTextOP(key) {
  const inputObj = {
    translateText: {
      sourceLanguage: 'en',
      targetLanguage: 'es'
    },
    identifyText: { key },
    convertTextToSpeech: { voiceID: 'Conchita' }
  };
  const response = await client.graphql({
    query: speakTranslatedImageText,
    variables: { input: inputObj }
  });
  return response.data.speakTranslatedImageText;
}

function App() {
  return (
    <div className="App">
      <h1>Speak Translated Image</h1>
      <SpeakTranslatedImage />
    </div>
  );
}
export default App;

directive @predictions(actions: [PredictionsActions!]!) on FIELD_DEFINITION
enum PredictionsActions {
  identifyText # uses Amazon Rekognition to detect text
  identifyLabels # uses Amazon Rekognition to detect labels
  convertTextToSpeech # uses Amazon Polly in a lambda to output a presigned url to synthesized speech
  translateText # uses Amazon Translate to translate text from source to target language
}