Amazon AppStream
Developer Guide

This documentation is for an older version of Amazon AppStream. For information about the latest version, see the Amazon AppStream 2.0 Developer Guide.

Sending Your Client Inputs to the Application

The client can send key, and touch inputs to the application.

The following excerpt from the example client in the Amazon AppStream SDK illustrates how keyboard events are handled. The excerpt is from the setupInputs function of AppStreamSampleClientViewController.m.

-(void) setupInputs { // KEYBOARD EVENTS -- pass through iOS key presses CGFloat y = 30.0f; // add a button for keyboard input UIButton *kbbutton = [UIButton buttonWithType:UIButtonTypeCustom]; [kbbutton addTarget:self action:@selector(toggleKeyboard:) forControlEvents:UIControlEventTouchUpInside]; CGFloat buttonWidth = 72.0f; kbbutton.frame = CGRectMake(self.view.bounds.size.width-buttonWidth, y,buttonWidth,buttonWidth); [kbbutton setImage:[UIImage imageNamed:@"button-keyboard"] forState:UIControlStateNormal]; kbbutton.autoresizingMask = UIViewAutoresizingFlexibleBottomMargin|UIViewAutoresizingFlexibleLeftMargin; [self.view addSubview:kbbutton]; // add offscreen text field for keyboard input and set delegate // note: text field is off screen to hide input from user mInputText = [[UITextField alloc]initWithFrame:CGRectMake(-500,-500,100,20)]; [self.view addSubview:mInputText]; mInputText.delegate = self; /// SENDING TOUCH AS MOUSE EVENTS mTreatTouchesAsMouse = NO; // add a button for mouse input UIButton *mousebutton = [UIButton buttonWithType:UIButtonTypeCustom]; [mousebutton addTarget:self action:@selector(toggleMouse:) forControlEvents:UIControlEventTouchUpInside]; mousebutton.frame = CGRectMake(self.view.bounds.size.width-(buttonWidth*2), y,buttonWidth,buttonWidth); [mousebutton setImage:[UIImage imageNamed:@"button-mouse"] forState:UIControlStateNormal]; mousebutton.autoresizingMask = UIViewAutoresizingFlexibleBottomMargin|UIViewAutoresizingFlexibleLeftMargin; [self.view addSubview:mousebutton]; mShouldTrackGesture = false; UIButton *handbutton = [UIButton buttonWithType:UIButtonTypeCustom]; [handbutton addTarget:self action:@selector(toggleGesture:) forControlEvents:UIControlEventTouchUpInside]; handbutton.frame = CGRectMake(self.view.bounds.size.width-(buttonWidth*3), y,buttonWidth,buttonWidth); [handbutton setImage:[UIImage imageNamed:@"button-hand"] forState:UIControlStateNormal]; handbutton.autoresizingMask = UIViewAutoresizingFlexibleBottomMargin|UIViewAutoresizingFlexibleLeftMargin; [self.view addSubview:handbutton]; mPanGestureRecognizer = [[UIPanGestureRecognizer alloc]initWithTarget:self action:@selector(handlePanGesture:)]; mPanGestureRecognizer.enabled = false; [self.view addGestureRecognizer:mPanGestureRecognizer]; }

The following excerpt from the example client in the Amazon AppStream SDK illustrates how gesture events are handled. The excerpt is from the handlePanGesture function of AppStreamSampleClientViewController.m.

void) handlePanGesture:(UIPanGestureRecognizer*) gestureRecognizer { CGPoint location = [gestureRecognizer locationInView:self.view]; NSString *jsonString = [NSString stringWithFormat:@"{\"state\":\"%d\",\"xy\":[%.2f,%.2f]}",gestureRecognizer.state,location.x, location.y ]; switch (gestureRecognizer.state) { case UIGestureRecognizerStateEnded: [self showStatus:@""]; break; case UIGestureRecognizerStatePossible: case UIGestureRecognizerStateBegan: case UIGestureRecognizerStateChanged: case UIGestureRecognizerStateFailed: case UIGestureRecognizerStateCancelled: default: [self sendRawInput:jsonString]; break; } }

Touch events are handled as mouse events.

The following excerpt from the example client in the Amazon AppStream SDK illustrates this step. The excerpt is from AppStreamSampleClientViewController.m.

// In this example we send the touch events as mouse input -(void) toggleMouse:(UIButton*)sender { if(mShouldTrackGesture) { mShouldTrackGesture = false; mPanGestureRecognizer.enabled = false; mTreatTouchesAsMouse = true; } else { if( mTreatTouchesAsMouse) { mTreatTouchesAsMouse = false; [self showStatus:@"touches passed as touches"]; } else { mTreatTouchesAsMouse = true; [self showStatus:@"mouse input emulated by touch "]; } } } #pragma mark UIResponder touch methods -(void) touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { if (!mTreatTouchesAsMouse) { [self sendTouchEvent: XSTX_TOUCH_DOWN withTouches: touches withEvent: event]; return; } UITouch *touch = [touches anyObject]; [self sendMouseEvent:[touch locationInView:self.view] flags:RI_MOUSE_BUTTON_1_DOWN]; } -(void) touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event { //When passing touches as touches we need to separate out touch_cancelled if (!mTreatTouchesAsMouse) { [self sendTouchEvent: XSTX_TOUCH_CANCELLED withTouches: touches withEvent: event]; return; } //Handle touchesCancelled exactly like touchesEnded [self touchesEnded:touches withEvent:event]; } -(void) touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event { if (!mTreatTouchesAsMouse) { [self sendTouchEvent: XSTX_TOUCH_UP withTouches: touches withEvent: event]; return; } UITouch *touch = [touches anyObject]; [self sendMouseEvent:[touch locationInView:self.view] flags:RI_MOUSE_BUTTON_1_UP]; //After the mouseUp move the mouse way offscreen to prevent any unwanted mouse hover effects [self sendMouseEvent:CGPointMake(-10000, -10000) flags:0]; // clear status [self showStatus:@""]; } -(void) touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event { if (!mTreatTouchesAsMouse) { [self sendTouchEvent: XSTX_TOUCH_MOVE withTouches: touches withEvent: event]; return; } UITouch *touch = [touches anyObject]; [self sendMouseEvent:[touch locationInView:self.view] flags:0]; } #pragma mark XStx touch event handling -(void) sendTouchEvent: (XStxTouchType) theType withTouches: (NSSet *) touches withEvent: (UIEvent *) event { if (mWidthScale==0) { return; } // here we are just going to emulate the mouse as a touch event, // just to demonstrate how to use it XStxInputEvent inputEvent; inputEvent.mTimestampUs = CACurrentMediaTime() * 1000000.0; inputEvent.mUserId = 0; inputEvent.mType = XSTX_INPUT_EVENT_TYPE_TOUCH; //How many touches we are sending inputEvent.mInfo.mTouch.mPointerCount = [touches count]; //Array to hold the pointer data inputEvent.mInfo.mTouch.mPointers = new XStxPointer[[touches count]]; int currPointer = 0; for (UITouch *currTouch in touches) { //Convert the touch location from the view to the server coordinates CGPoint touchLoc = [currTouch locationInView:self.view]; //Make sure touchLoc is within the viewport area touchLoc = CGPointMake(MIN(MAX(touchLoc.x - mVideoRect.origin.x, 0), mVideoRect.size.width), MIN(MAX(touchLoc.y - mVideoRect.origin.y, 0), mVideoRect.size.height)); float scaledX = (touchLoc.x * mWidthScale); float scaledY = (touchLoc.y * mHeightScale); //Use the address of the touchObject as the pointerID inputEvent.mInfo.mTouch.mPointers[currPointer].mPointerId = (uint64_t)currTouch; //Set the X & Y inputEvent.mInfo.mTouch.mPointers[currPointer].mX = scaledX; inputEvent.mInfo.mTouch.mPointers[currPointer].mY = scaledY; //No pressure support so always pass 1.0 inputEvent.mInfo.mTouch.mPointers[currPointer].mPressure = 1.0f; inputEvent.mInfo.mTouch.mPointers[currPointer].mTouchType = theType; currPointer++; } inputEvent.mSize = sizeof(inputEvent); inputEvent.mDeviceId = 0; //Send the touches to the server [self sendInput:inputEvent]; //Clean up the mPointers array delete [] inputEvent.mInfo.mTouch.mPointers; } // fill out XStxInputEvent struct emulating win32 RAWMOUSE -(void) sendMouseEvent:(CGPoint ) xy flags:(uint32_t)flags { if (mWidthScale==0) { return; } //Make sure xy is within the viewport area xy = CGPointMake(MIN(MAX(xy.x - mVideoRect.origin.x, 0), mVideoRect.size.width), MIN(MAX(xy.y - mVideoRect.origin.y, 0), mVideoRect.size.height)); int scaledX = (xy.x * mWidthScale); int scaledY = (xy.y * mHeightScale); XStxInputEvent xstxevent = { 0 }; xstxevent.mTimestampUs = CACurrentMediaTime() * 1000000.0; xstxevent.mType = XSTX_INPUT_EVENT_TYPE_MOUSE; xstxevent.mInfo.mMouse.mLastX = scaledX; xstxevent.mInfo.mMouse.mLastY = scaledY; xstxevent.mInfo.mMouse.mButtonFlags = flags ; xstxevent.mInfo.mMouse.mFlags = 1; //absolute, 0 would be relative xstxevent.mInfo.mMouse.mButtons = 0;// not needed [self sendInput:xstxevent]; NSString *outputString = [NSString stringWithFormat:@"sending mouse event with coordinates:x:%i y:%i",scaledX,scaledY]; [self showStatus:outputString]; }