Source/WebDriver/Actions.h

 1/*
 2 * Copyright (C) 2018 Igalia S.L.
 3 *
 4 * Redistribution and use in source and binary forms, with or without
 5 * modification, are permitted provided that the following conditions
 6 * are met:
 7 * 1. Redistributions of source code must retain the above copyright
 8 * notice, this list of conditions and the following disclaimer.
 9 * 2. Redistributions in binary form must reproduce the above copyright
 10 * notice, this list of conditions and the following disclaimer in the
 11 * documentation and/or other materials provided with the distribution.
 12 *
 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 23 * THE POSSIBILITY OF SUCH DAMAGE.
 24 */
 25
 26#pragma once
 27
 28#include <wtf/text/WTFString.h>
 29
 30namespace WebDriver {
 31
 32enum class MouseButton { None, Left, Middle, Right };
 33enum class PointerType { Mouse, Pen, Touch };
 34
 35struct InputSource {
 36 enum class Type { None, Key, Pointer };
 37
 38 Type type;
 39 std::optional<PointerType> pointerType;
 40};
 41
 42struct PointerParameters {
 43 PointerType pointerType { PointerType::Mouse };
 44};
 45
 46struct PointerOrigin {
 47 enum class Type { Viewport, Pointer, Element };
 48
 49 Type type;
 50 std::optional<String> elementID;
 51};
 52
 53struct Action {
 54 enum class Type { None, Key, Pointer };
 55 enum class Subtype { Pause, PointerUp, PointerDown, PointerMove, PointerCancel, KeyUp, KeyDown };
 56
 57 Action(const String& id, Type type, Subtype subtype)
 58 : id(id)
 59 , type(type)
 60 , subtype(subtype)
 61 {
 62 }
 63
 64 String id;
 65 Type type;
 66 Subtype subtype;
 67 std::optional<unsigned> duration;
 68
 69 std::optional<PointerType> pointerType;
 70 std::optional<MouseButton> button;
 71 std::optional<PointerOrigin> origin;
 72 std::optional<int64_t> x;
 73 std::optional<int64_t> y;
 74
 75 std::optional<String> key;
 76};
 77
 78} // WebDriver

Source/WebDriver/ChangeLog

 12018-05-09 Carlos Garcia Campos <cgarcia@igalia.com>
 2
 3 WebDriver: implement advance user interactions
 4 https://bugs.webkit.org/show_bug.cgi?id=174616
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 Add initial implementation of action commands.
 9
 10 * Actions.h: Added.
 11 (WebDriver::Action::Action):
 12 * CommandResult.cpp:
 13 (WebDriver::CommandResult::CommandResult): Handle MoveTargetOutOfBounds error.
 14 (WebDriver::CommandResult::httpStatusCode const): Ditto.
 15 (WebDriver::CommandResult::errorString const): Ditto.
 16 * CommandResult.h:
 17 * Session.cpp:
 18 (WebDriver::Session::webElementIdentifier): Helper to return the web element id.
 19 (WebDriver::Session::createElement): Use webElementIdentifier().
 20 (WebDriver::Session::extractElementID): Ditto.
 21 (WebDriver::Session::virtualKeyForKeySequence): Add more kay codes includes in the spec.
 22 (WebDriver::mouseButtonForAutomation): Helper to get the mouse button string to pass to automation.
 23 (WebDriver::Session::performMouseInteraction): Use mouseButtonForAutomation().
 24 (WebDriver::Session::getOrCreateInputSource): Ensure an input source for given id and add it to the active input
 25 sources.
 26 (WebDriver::Session::inputSourceState): Return the current input source state for the given id.
 27 (WebDriver::Session::computeInViewCenterPointOfElements): Get the in view center point for the list of elements given.
 28 (WebDriver::automationSourceType): Helper to get the input source type to pass to automation.
 29 (WebDriver::Session::performActions): Process the list of action by tick and generate a list of states to pass
 30 to automation.
 31 (WebDriver::Session::releaseActions): Reset input sources and state table and send a message to automation.
 32 * Session.h:
 33 * WebDriverService.cpp:
 34 (WebDriver::processPauseAction):
 35 (WebDriver::processNullAction):
 36 (WebDriver::processKeyAction):
 37 (WebDriver::actionMouseButton):
 38 (WebDriver::processPointerAction):
 39 (WebDriver::processPointerParameters):
 40 (WebDriver::processInputActionSequence):
 41 (WebDriver::WebDriverService::performActions):
 42 (WebDriver::WebDriverService::releaseActions):
 43 * WebDriverService.h:
 44
1452018-03-05 Carlos Garcia Campos <cgarcia@igalia.com>
246
347 WebDriver: Also ignore NoSuchwindow errors when waiting for navigation to complete

Source/WebDriver/CommandResult.cpp

@@CommandResult::CommandResult(RefPtr<JSON::Value>&& result, std::optional<ErrorCo
108108 m_errorCode = ErrorCode::UnableToCaptureScreen;
109109 else if (errorName == "UnexpectedAlertOpen")
110110 m_errorCode = ErrorCode::UnexpectedAlertOpen;
 111 else if (errorName == "TargetOutOfBounds")
 112 m_errorCode = ErrorCode::MoveTargetOutOfBounds;
111113
112114 break;
113115 }

@@unsigned CommandResult::httpStatusCode() const
148150 case ErrorCode::Timeout:
149151 return 408;
150152 case ErrorCode::JavascriptError:
 153 case ErrorCode::MoveTargetOutOfBounds:
151154 case ErrorCode::SessionNotCreated:
152155 case ErrorCode::UnableToCaptureScreen:
153156 case ErrorCode::UnexpectedAlertOpen:

@@String CommandResult::errorString() const
201204 return ASCIILiteral("timeout");
202205 case ErrorCode::UnableToCaptureScreen:
203206 return ASCIILiteral("unable to capture screen");
 207 case ErrorCode::MoveTargetOutOfBounds:
 208 return ASCIILiteral("move target out of bounds");
204209 case ErrorCode::UnexpectedAlertOpen:
205210 return ASCIILiteral("unexpected alert open");
206211 case ErrorCode::UnknownCommand:

Source/WebDriver/CommandResult.h

@@public:
4444 InvalidSelector,
4545 InvalidSessionID,
4646 JavascriptError,
 47 MoveTargetOutOfBounds,
4748 NoSuchAlert,
4849 NoSuchCookie,
4950 NoSuchElement,

Source/WebDriver/Session.cpp

3030#include "SessionHost.h"
3131#include "WebDriverAtoms.h"
3232#include <wtf/CryptographicallyRandomNumber.h>
 33#include <wtf/HashSet.h>
3334#include <wtf/HexNumber.h>
 35#include <wtf/NeverDestroyed.h>
3436
3537namespace WebDriver {
3638
37 // The web element identifier is a constant defined by the spec in Section 11 Elements.
38 // https://www.w3.org/TR/webdriver/#elements
39 static const String webElementIdentifier = ASCIILiteral("element-6066-11e4-a52e-4f735466cecf");
40 
4139// https://w3c.github.io/webdriver/webdriver-spec.html#dfn-session-script-timeout
4240static const Seconds defaultScriptTimeout = 30_s;
4341// https://w3c.github.io/webdriver/webdriver-spec.html#dfn-session-page-load-timeout

@@static const Seconds defaultPageLoadTimeout = 300_s;
4543// https://w3c.github.io/webdriver/webdriver-spec.html#dfn-session-implicit-wait-timeout
4644static const Seconds defaultImplicitWaitTimeout = 0_s;
4745
 46const String& Session::webElementIdentifier()
 47{
 48 // The web element identifier is a constant defined by the spec in Section 11 Elements.
 49 // https://www.w3.org/TR/webdriver/#elements
 50 static NeverDestroyed<String> webElementID { ASCIILiteral("element-6066-11e4-a52e-4f735466cecf") };
 51 return webElementID;
 52}
 53
4854Session::Session(std::unique_ptr<SessionHost>&& host)
4955 : m_host(WTFMove(host))
5056 , m_scriptTimeout(defaultScriptTimeout)

@@RefPtr<JSON::Object> Session::createElement(RefPtr<JSON::Value>&& value)
779785 return nullptr;
780786
781787 RefPtr<JSON::Object> elementObject = JSON::Object::create();
782  elementObject->setString(webElementIdentifier, elementID);
 788 elementObject->setString(webElementIdentifier(), elementID);
783789 return elementObject;
784790}
785791

@@String Session::extractElementID(JSON::Value& value)
803809 return emptyString();
804810
805811 String elementID;
806  if (!valueObject->getString(webElementIdentifier, elementID))
 812 if (!valueObject->getString(webElementIdentifier(), elementID))
807813 return emptyString();
808814
809815 return elementID;

@@String Session::virtualKeyForKeySequence(const String& keySequence, KeyModifier&
15041510 case 0xE007U:
15051511 return ASCIILiteral("Enter");
15061512 case 0xE008U:
 1513 case 0xE050U:
15071514 modifier = KeyModifier::Shift;
15081515 return ASCIILiteral("Shift");
15091516 case 0xE009U:
 1517 case 0xE051U:
15101518 modifier = KeyModifier::Control;
15111519 return ASCIILiteral("Control");
15121520 case 0xE00AU:
 1521 case 0xE052U:
15131522 modifier = KeyModifier::Alternate;
15141523 return ASCIILiteral("Alternate");
15151524 case 0xE00BU:

@@String Session::virtualKeyForKeySequence(const String& keySequence, KeyModifier&
15191528 case 0xE00DU:
15201529 return ASCIILiteral("Space");
15211530 case 0xE00EU:
 1531 case 0xE054U:
15221532 return ASCIILiteral("PageUp");
15231533 case 0xE00FU:
 1534 case 0xE055U:
15241535 return ASCIILiteral("PageDown");
15251536 case 0xE010U:
 1537 case 0xE056U:
15261538 return ASCIILiteral("End");
15271539 case 0xE011U:
 1540 case 0xE057U:
15281541 return ASCIILiteral("Home");
15291542 case 0xE012U:
 1543 case 0xE058U:
15301544 return ASCIILiteral("LeftArrow");
15311545 case 0xE013U:
 1546 case 0xE059U:
15321547 return ASCIILiteral("UpArrow");
15331548 case 0xE014U:
 1549 case 0xE05AU:
15341550 return ASCIILiteral("RightArrow");
15351551 case 0xE015U:
 1552 case 0xE05BU:
15361553 return ASCIILiteral("DownArrow");
15371554 case 0xE016U:
 1555 case 0xE05CU:
15381556 return ASCIILiteral("Insert");
15391557 case 0xE017U:
 1558 case 0xE05DU:
15401559 return ASCIILiteral("Delete");
15411560 case 0xE018U:
15421561 return ASCIILiteral("Semicolon");

@@String Session::virtualKeyForKeySequence(const String& keySequence, KeyModifier&
15991618 case 0xE03CU:
16001619 return ASCIILiteral("Function12");
16011620 case 0xE03DU:
 1621 case 0xE053U:
16021622 modifier = KeyModifier::Meta;
16031623 return ASCIILiteral("Meta");
16041624 default:

@@void Session::executeScript(const String& script, RefPtr<JSON::Array>&& argument
17691789 });
17701790}
17711791
 1792static String mouseButtonForAutomation(MouseButton button)
 1793{
 1794 switch (button) {
 1795 case MouseButton::None:
 1796 return ASCIILiteral("None");
 1797 case MouseButton::Left:
 1798 return ASCIILiteral("Left");
 1799 case MouseButton::Middle:
 1800 return ASCIILiteral("Middle");
 1801 case MouseButton::Right:
 1802 return ASCIILiteral("Right");
 1803 }
 1804
 1805 RELEASE_ASSERT_NOT_REACHED();
 1806}
 1807
17721808void Session::performMouseInteraction(int x, int y, MouseButton button, MouseInteraction interaction, Function<void (CommandResult&&)>&& completionHandler)
17731809{
17741810 RefPtr<JSON::Object> parameters = JSON::Object::create();

@@void Session::performMouseInteraction(int x, int y, MouseButton button, MouseInt
17771813 position->setInteger(ASCIILiteral("x"), x);
17781814 position->setInteger(ASCIILiteral("y"), y);
17791815 parameters->setObject(ASCIILiteral("position"), WTFMove(position));
1780  switch (button) {
1781  case MouseButton::None:
1782  parameters->setString(ASCIILiteral("button"), ASCIILiteral("None"));
1783  break;
1784  case MouseButton::Left:
1785  parameters->setString(ASCIILiteral("button"), ASCIILiteral("Left"));
1786  break;
1787  case MouseButton::Middle:
1788  parameters->setString(ASCIILiteral("button"), ASCIILiteral("Middle"));
1789  break;
1790  case MouseButton::Right:
1791  parameters->setString(ASCIILiteral("button"), ASCIILiteral("Right"));
1792  break;
1793  }
 1816 parameters->setString(ASCIILiteral("button"), mouseButtonForAutomation(button));
17941817 switch (interaction) {
17951818 case MouseInteraction::Move:
17961819 parameters->setString(ASCIILiteral("interaction"), ASCIILiteral("Move"));

@@void Session::deleteAllCookies(Function<void (CommandResult&&)>&& completionHand
20592082 });
20602083}
20612084
 2085InputSource& Session::getOrCreateInputSource(const String& id, InputSource::Type type, std::optional<PointerType> pointerType)
 2086{
 2087 auto addResult = m_activeInputSources.add(id, InputSource());
 2088 if (addResult.isNewEntry)
 2089 addResult.iterator->value = { type, pointerType };
 2090 return addResult.iterator->value;
 2091}
 2092
 2093Session::InputSourceState& Session::inputSourceState(const String& id)
 2094{
 2095 return m_inputStateTable.ensure(id, [] { return InputSourceState(); }).iterator->value;
 2096}
 2097
 2098static const char* automationSourceType(InputSource::Type type)
 2099{
 2100 switch (type) {
 2101 case InputSource::Type::None:
 2102 return "Null";
 2103 case InputSource::Type::Pointer:
 2104 return "Mouse";
 2105 case InputSource::Type::Key:
 2106 return "Keyboard";
 2107 }
 2108 RELEASE_ASSERT_NOT_REACHED();
 2109}
 2110
 2111static const char* automationOriginType(PointerOrigin::Type type)
 2112{
 2113 switch (type) {
 2114 case PointerOrigin::Type::Viewport:
 2115 return "Viewport";
 2116 case PointerOrigin::Type::Pointer:
 2117 return "Pointer";
 2118 case PointerOrigin::Type::Element:
 2119 return "Element";
 2120 }
 2121 RELEASE_ASSERT_NOT_REACHED();
 2122}
 2123
 2124void Session::performActions(Vector<Vector<Action>>&& actionsByTick, Function<void (CommandResult&&)>&& completionHandler)
 2125{
 2126 if (!m_toplevelBrowsingContext) {
 2127 completionHandler(CommandResult::fail(CommandResult::ErrorCode::NoSuchWindow));
 2128 return;
 2129 }
 2130
 2131 handleUserPrompts([this, actionsByTick = WTFMove(actionsByTick), completionHandler = WTFMove(completionHandler)](CommandResult&& result) mutable {
 2132 if (result.isError()) {
 2133 completionHandler(WTFMove(result));
 2134 return;
 2135 }
 2136
 2137 // First check if we have actions and whether we need to resolve any pointer move element origin.
 2138 unsigned actionsCount = 0;
 2139 for (const auto& tick : actionsByTick)
 2140 actionsCount += tick.size();
 2141 if (!actionsCount) {
 2142 completionHandler(CommandResult::success());
 2143 return;
 2144 }
 2145
 2146 RefPtr<JSON::Object> parameters = JSON::Object::create();
 2147 parameters->setString(ASCIILiteral("handle"), m_toplevelBrowsingContext.value());
 2148 if (m_currentBrowsingContext)
 2149 parameters->setString(ASCIILiteral("frameHandle"), m_currentBrowsingContext.value());
 2150 RefPtr<JSON::Array> inputSources = JSON::Array::create();
 2151 for (const auto& inputSource : m_activeInputSources) {
 2152 RefPtr<JSON::Object> inputSourceObject = JSON::Object::create();
 2153 inputSourceObject->setString(ASCIILiteral("sourceId"), inputSource.key);
 2154 inputSourceObject->setString(ASCIILiteral("sourceType"), automationSourceType(inputSource.value.type));
 2155 inputSources->pushObject(WTFMove(inputSourceObject));
 2156 }
 2157 parameters->setArray(ASCIILiteral("inputSources"), WTFMove(inputSources));
 2158 RefPtr<JSON::Array> steps = JSON::Array::create();
 2159 for (const auto& tick : actionsByTick) {
 2160 RefPtr<JSON::Array> states = JSON::Array::create();
 2161 for (const auto& action : tick) {
 2162 RefPtr<JSON::Object> state = JSON::Object::create();
 2163 auto& currentState = inputSourceState(action.id);
 2164 state->setString(ASCIILiteral("sourceId"), action.id);
 2165 switch (action.type) {
 2166 case Action::Type::None:
 2167 state->setDouble(ASCIILiteral("duration"), action.duration.value());
 2168 break;
 2169 case Action::Type::Pointer: {
 2170 switch (action.subtype) {
 2171 case Action::Subtype::PointerUp:
 2172 currentState.pressedButton = std::nullopt;
 2173 break;
 2174 case Action::Subtype::PointerDown:
 2175 currentState.pressedButton = action.button.value();
 2176 break;
 2177 case Action::Subtype::PointerMove: {
 2178 state->setString(ASCIILiteral("origin"), automationOriginType(action.origin->type));
 2179 RefPtr<JSON::Object> location = JSON::Object::create();
 2180 location->setInteger(ASCIILiteral("x"), action.x.value());
 2181 location->setInteger(ASCIILiteral("y"), action.y.value());
 2182 state->setObject(ASCIILiteral("location"), WTFMove(location));
 2183 if (action.origin->type == PointerOrigin::Type::Element)
 2184 state->setString(ASCIILiteral("nodeHandle"), action.origin->elementID.value());
 2185 FALLTHROUGH;
 2186 }
 2187 case Action::Subtype::Pause:
 2188 if (action.duration)
 2189 state->setDouble(ASCIILiteral("duration"), action.duration.value());
 2190 break;
 2191 case Action::Subtype::PointerCancel:
 2192 currentState.pressedButton = std::nullopt;
 2193 break;
 2194 case Action::Subtype::KeyUp:
 2195 case Action::Subtype::KeyDown:
 2196 ASSERT_NOT_REACHED();
 2197 }
 2198 if (currentState.pressedButton)
 2199 state->setString(ASCIILiteral("pressedButton"), mouseButtonForAutomation(currentState.pressedButton.value()));
 2200 break;
 2201 }
 2202 case Action::Type::Key:
 2203 switch (action.subtype) {
 2204 case Action::Subtype::KeyUp:
 2205 if (currentState.pressedVirtualKey)
 2206 currentState.pressedVirtualKey = std::nullopt;
 2207 else
 2208 currentState.pressedKey = std::nullopt;
 2209 break;
 2210 case Action::Subtype::KeyDown: {
 2211 KeyModifier modifier;
 2212 auto virtualKey = virtualKeyForKeySequence(action.key.value(), modifier);
 2213 if (!virtualKey.isNull())
 2214 currentState.pressedVirtualKey = virtualKey;
 2215 else
 2216 currentState.pressedKey = action.key.value();
 2217 break;
 2218 }
 2219 case Action::Subtype::Pause:
 2220 if (action.duration)
 2221 state->setDouble(ASCIILiteral("duration"), action.duration.value());
 2222 break;
 2223 case Action::Subtype::PointerUp:
 2224 case Action::Subtype::PointerDown:
 2225 case Action::Subtype::PointerMove:
 2226 case Action::Subtype::PointerCancel:
 2227 ASSERT_NOT_REACHED();
 2228 }
 2229 if (currentState.pressedKey)
 2230 state->setString(ASCIILiteral("pressedCharKey"), currentState.pressedKey.value());
 2231 if (currentState.pressedVirtualKey)
 2232 state->setString(ASCIILiteral("pressedVirtualKey"), currentState.pressedVirtualKey.value());
 2233 break;
 2234 }
 2235 states->pushObject(WTFMove(state));
 2236 }
 2237 RefPtr<JSON::Object> stepStates = JSON::Object::create();
 2238 stepStates->setArray(ASCIILiteral("states"), WTFMove(states));
 2239 steps->pushObject(WTFMove(stepStates));
 2240 }
 2241
 2242 parameters->setArray(ASCIILiteral("steps"), WTFMove(steps));
 2243 m_host->sendCommandToBackend(ASCIILiteral("performInteractionSequence"), WTFMove(parameters), [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)] (SessionHost::CommandResponse&& response) {
 2244 if (response.isError) {
 2245 completionHandler(CommandResult::fail(WTFMove(response.responseObject)));
 2246 return;
 2247 }
 2248 completionHandler(CommandResult::success());
 2249 });
 2250 });
 2251}
 2252
 2253void Session::releaseActions(Function<void (CommandResult&&)>&& completionHandler)
 2254{
 2255 if (!m_toplevelBrowsingContext) {
 2256 completionHandler(CommandResult::fail(CommandResult::ErrorCode::NoSuchWindow));
 2257 return;
 2258 }
 2259
 2260 m_activeInputSources.clear();
 2261 m_inputStateTable.clear();
 2262
 2263 RefPtr<JSON::Object> parameters = JSON::Object::create();
 2264 parameters->setString(ASCIILiteral("handle"), m_toplevelBrowsingContext.value());
 2265 m_host->sendCommandToBackend(ASCIILiteral("cancelInteractionSequence"), WTFMove(parameters), [this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler)](SessionHost::CommandResponse&& response) {
 2266 if (response.isError) {
 2267 completionHandler(CommandResult::fail(WTFMove(response.responseObject)));
 2268 return;
 2269 }
 2270 completionHandler(CommandResult::success());
 2271 });
 2272}
 2273
20622274void Session::dismissAlert(Function<void (CommandResult&&)>&& completionHandler)
20632275{
20642276 if (!m_toplevelBrowsingContext) {

Source/WebDriver/Session.h

2525
2626#pragma once
2727
 28#include "Actions.h"
2829#include "Capabilities.h"
2930#include <wtf/Forward.h>
3031#include <wtf/Function.h>

@@public:
5253 Seconds scriptTimeout() const { return m_scriptTimeout; }
5354 Seconds pageLoadTimeout() const { return m_pageLoadTimeout; }
5455 Seconds implicitWaitTimeout() const { return m_implicitWaitTimeout; }
 56 static const String& webElementIdentifier();
5557
5658 enum class FindElementsMode { Single, Multiple };
5759 enum class ExecuteScriptMode { Sync, Async };

@@public:
6668 std::optional<uint64_t> expiry;
6769 };
6870
 71 InputSource& getOrCreateInputSource(const String& id, InputSource::Type, std::optional<PointerType>);
 72
6973 void waitForNavigationToComplete(Function<void (CommandResult&&)>&&);
7074 void createTopLevelBrowsingContext(Function<void (CommandResult&&)>&&);
7175 void close(Function<void (CommandResult&&)>&&);

@@public:
106110 void addCookie(const Cookie&, Function<void (CommandResult&&)>&&);
107111 void deleteCookie(const String& name, Function<void (CommandResult&&)>&&);
108112 void deleteAllCookies(Function<void (CommandResult&&)>&&);
 113 void performActions(Vector<Vector<Action>>&&, Function<void (CommandResult&&)>&&);
 114 void releaseActions(Function<void (CommandResult&&)>&&);
109115 void dismissAlert(Function<void (CommandResult&&)>&&);
110116 void acceptAlert(Function<void (CommandResult&&)>&&);
111117 void getAlertText(Function<void (CommandResult&&)>&&);

@@private:
159165
160166 void selectOptionElement(const String& elementID, Function<void (CommandResult&&)>&&);
161167
162  enum class MouseButton { None, Left, Middle, Right };
163168 enum class MouseInteraction { Move, Down, Up, SingleClick, DoubleClick };
164169 void performMouseInteraction(int x, int y, MouseButton, MouseInteraction, Function<void (CommandResult&&)>&&);
165170

@@private:
179184 String virtualKeyForKeySequence(const String& keySequence, KeyModifier&);
180185 void performKeyboardInteractions(Vector<KeyboardInteraction>&&, Function<void (CommandResult&&)>&&);
181186
 187 struct InputSourceState {
 188 enum class Type { Null, Key, Pointer };
 189
 190 Type type;
 191 String subtype;
 192 std::optional<MouseButton> pressedButton;
 193 std::optional<String> pressedKey;
 194 std::optional<String> pressedVirtualKey;
 195 };
 196 InputSourceState& inputSourceState(const String& id);
 197
182198 std::unique_ptr<SessionHost> m_host;
183199 Seconds m_scriptTimeout;
184200 Seconds m_pageLoadTimeout;
185201 Seconds m_implicitWaitTimeout;
186202 std::optional<String> m_toplevelBrowsingContext;
187203 std::optional<String> m_currentBrowsingContext;
 204 HashMap<String, InputSource> m_activeInputSources;
 205 HashMap<String, InputSourceState> m_inputStateTable;
188206};
189207
190208} // WebDriver

Source/WebDriver/WebDriverService.cpp

@@const WebDriverService::Command WebDriverService::s_commands[] = {
151151 { HTTPMethod::Delete, "/session/$sessionId/cookie/$name", &WebDriverService::deleteCookie },
152152 { HTTPMethod::Delete, "/session/$sessionId/cookie", &WebDriverService::deleteAllCookies },
153153
 154 { HTTPMethod::Post, "/session/$sessionId/actions", &WebDriverService::performActions },
 155 { HTTPMethod::Delete, "/session/$sessionId/actions", &WebDriverService::releaseActions },
 156
154157 { HTTPMethod::Post, "/session/$sessionId/alert/dismiss", &WebDriverService::dismissAlert },
155158 { HTTPMethod::Post, "/session/$sessionId/alert/accept", &WebDriverService::acceptAlert },
156159 { HTTPMethod::Get, "/session/$sessionId/alert/text", &WebDriverService::getAlertText },

@@void WebDriverService::deleteAllCookies(RefPtr<JSON::Object>&& parameters, Funct
15871590 });
15881591}
15891592
 1593static bool processPauseAction(JSON::Object& actionItem, Action& action, std::optional<String>& errorMessage)
 1594{
 1595 RefPtr<JSON::Value> durationValue;
 1596 if (!actionItem.getValue(ASCIILiteral("duration"), durationValue)) {
 1597 errorMessage = String("The parameter 'duration' is missing in pause action");
 1598 return false;
 1599 }
 1600
 1601 auto duration = unsignedValue(*durationValue);
 1602 if (!duration) {
 1603 errorMessage = String("The parameter 'duration' is invalid in pause action");
 1604 return false;
 1605 }
 1606
 1607 action.duration = duration.value();
 1608 return true;
 1609}
 1610
 1611static std::optional<Action> processNullAction(const String& id, JSON::Object& actionItem, std::optional<String>& errorMessage)
 1612{
 1613 String subtype;
 1614 actionItem.getString(ASCIILiteral("type"), subtype);
 1615 if (subtype != "pause") {
 1616 errorMessage = String("The parameter 'type' in null action is invalid or missing");
 1617 return std::nullopt;
 1618 }
 1619
 1620 Action action(id, Action::Type::None, Action::Subtype::Pause);
 1621 if (!processPauseAction(actionItem, action, errorMessage))
 1622 return std::nullopt;
 1623
 1624 return action;
 1625}
 1626
 1627static std::optional<Action> processKeyAction(const String& id, JSON::Object& actionItem, std::optional<String>& errorMessage)
 1628{
 1629 Action::Subtype actionSubtype;
 1630 String subtype;
 1631 actionItem.getString(ASCIILiteral("type"), subtype);
 1632 if (subtype == "pause")
 1633 actionSubtype = Action::Subtype::Pause;
 1634 else if (subtype == "keyUp")
 1635 actionSubtype = Action::Subtype::KeyUp;
 1636 else if (subtype == "keyDown")
 1637 actionSubtype = Action::Subtype::KeyDown;
 1638 else {
 1639 errorMessage = String("The parameter 'type' of key action is invalid");
 1640 return std::nullopt;
 1641 }
 1642
 1643 Action action(id, Action::Type::Key, actionSubtype);
 1644
 1645 switch (actionSubtype) {
 1646 case Action::Subtype::Pause:
 1647 if (!processPauseAction(actionItem, action, errorMessage))
 1648 return std::nullopt;
 1649 break;
 1650 case Action::Subtype::KeyUp:
 1651 case Action::Subtype::KeyDown: {
 1652 RefPtr<JSON::Value> keyValue;
 1653 if (!actionItem.getValue(ASCIILiteral("value"), keyValue)) {
 1654 errorMessage = String("The paramater 'value' is missing for key up/down action");
 1655 return std::nullopt;
 1656 }
 1657 String key;
 1658 if (!keyValue->asString(key) || key.isEmpty()) {
 1659 errorMessage = String("The paramater 'value' is invalid for key up/down action");
 1660 return std::nullopt;
 1661 }
 1662 // FIXME: check single unicode code point.
 1663 action.key = key;
 1664 break;
 1665 }
 1666 case Action::Subtype::PointerUp:
 1667 case Action::Subtype::PointerDown:
 1668 case Action::Subtype::PointerMove:
 1669 case Action::Subtype::PointerCancel:
 1670 ASSERT_NOT_REACHED();
 1671 }
 1672
 1673 return action;
 1674}
 1675
 1676static MouseButton actionMouseButton(unsigned button)
 1677{
 1678 // MouseEvent.button
 1679 // https://www.w3.org/TR/uievents/#ref-for-dom-mouseevent-button-1
 1680 switch (button) {
 1681 case 0:
 1682 return MouseButton::Left;
 1683 case 1:
 1684 return MouseButton::Middle;
 1685 case 2:
 1686 return MouseButton::Right;
 1687 }
 1688
 1689 return MouseButton::None;
 1690}
 1691
 1692static std::optional<Action> processPointerAction(const String& id, PointerParameters& parameters, JSON::Object& actionItem, std::optional<String>& errorMessage)
 1693{
 1694 Action::Subtype actionSubtype;
 1695 String subtype;
 1696 actionItem.getString(ASCIILiteral("type"), subtype);
 1697 if (subtype == "pause")
 1698 actionSubtype = Action::Subtype::Pause;
 1699 else if (subtype == "pointerUp")
 1700 actionSubtype = Action::Subtype::PointerUp;
 1701 else if (subtype == "pointerDown")
 1702 actionSubtype = Action::Subtype::PointerDown;
 1703 else if (subtype == "pointerMove")
 1704 actionSubtype = Action::Subtype::PointerMove;
 1705 else if (subtype == "pointerCancel")
 1706 actionSubtype = Action::Subtype::PointerCancel;
 1707 else {
 1708 errorMessage = String("The parameter 'type' of pointer action is invalid");
 1709 return std::nullopt;
 1710 }
 1711
 1712 Action action(id, Action::Type::Pointer, actionSubtype);
 1713 action.pointerType = parameters.pointerType;
 1714
 1715 switch (actionSubtype) {
 1716 case Action::Subtype::Pause:
 1717 if (!processPauseAction(actionItem, action, errorMessage))
 1718 return std::nullopt;
 1719 break;
 1720 case Action::Subtype::PointerUp:
 1721 case Action::Subtype::PointerDown: {
 1722 RefPtr<JSON::Value> buttonValue;
 1723 if (!actionItem.getValue(ASCIILiteral("button"), buttonValue)) {
 1724 errorMessage = String("The paramater 'button' is missing for pointer up/down action");
 1725 return std::nullopt;
 1726 }
 1727 auto button = unsignedValue(*buttonValue);
 1728 if (!button) {
 1729 errorMessage = String("The paramater 'button' is invalid for pointer up/down action");
 1730 return std::nullopt;
 1731 }
 1732 action.button = actionMouseButton(button.value());
 1733 break;
 1734 }
 1735 case Action::Subtype::PointerMove: {
 1736 RefPtr<JSON::Value> durationValue;
 1737 if (actionItem.getValue(ASCIILiteral("duration"), durationValue)) {
 1738 auto duration = unsignedValue(*durationValue);
 1739 if (!duration) {
 1740 errorMessage = String("The parameter 'duration' is invalid in pointer move action");
 1741 return std::nullopt;
 1742 }
 1743 action.duration = duration.value();
 1744 }
 1745
 1746 RefPtr<JSON::Value> originValue;
 1747 if (actionItem.getValue(ASCIILiteral("origin"), originValue)) {
 1748 if (originValue->type() == JSON::Value::Type::Object) {
 1749 RefPtr<JSON::Object> originObject;
 1750 originValue->asObject(originObject);
 1751 String elementID;
 1752 if (!originObject->getString(Session::webElementIdentifier(), elementID)) {
 1753 errorMessage = String("The parameter 'origin' is not a valid web element object in pointer move action");
 1754 return std::nullopt;
 1755 }
 1756 action.origin = PointerOrigin { PointerOrigin::Type::Element, elementID };
 1757 } else {
 1758 String origin;
 1759 originValue->asString(origin);
 1760 if (origin == "viewport")
 1761 action.origin = PointerOrigin { PointerOrigin::Type::Viewport, std::nullopt };
 1762 else if (origin == "pointer")
 1763 action.origin = PointerOrigin { PointerOrigin::Type::Pointer, std::nullopt };
 1764 else {
 1765 errorMessage = String("The parameter 'origin' is invalid in pointer move action");
 1766 return std::nullopt;
 1767 }
 1768 }
 1769 } else
 1770 action.origin = PointerOrigin { PointerOrigin::Type::Viewport, std::nullopt };
 1771
 1772 RefPtr<JSON::Value> xValue;
 1773 if (actionItem.getValue(ASCIILiteral("x"), xValue)) {
 1774 auto x = valueAsNumberInRange(*xValue, INT_MIN);
 1775 if (!x) {
 1776 errorMessage = String("The paramater 'x' is invalid for pointer move action");
 1777 return std::nullopt;
 1778 }
 1779 action.x = x.value();
 1780 }
 1781
 1782 RefPtr<JSON::Value> yValue;
 1783 if (actionItem.getValue(ASCIILiteral("y"), yValue)) {
 1784 auto y = valueAsNumberInRange(*yValue, INT_MIN);
 1785 if (!y) {
 1786 errorMessage = String("The paramater 'y' is invalid for pointer move action");
 1787 return std::nullopt;
 1788 }
 1789 action.y = y.value();
 1790 }
 1791 break;
 1792 }
 1793 case Action::Subtype::PointerCancel:
 1794 break;
 1795 case Action::Subtype::KeyUp:
 1796 case Action::Subtype::KeyDown:
 1797 ASSERT_NOT_REACHED();
 1798 }
 1799
 1800 return action;
 1801}
 1802
 1803static std::optional<PointerParameters> processPointerParameters(JSON::Object& actionSequence, std::optional<String>& errorMessage)
 1804{
 1805 PointerParameters parameters;
 1806 RefPtr<JSON::Value> parametersDataValue;
 1807 if (!actionSequence.getValue(ASCIILiteral("parameters"), parametersDataValue))
 1808 return parameters;
 1809
 1810 RefPtr<JSON::Object> parametersData;
 1811 if (!parametersDataValue->asObject(parametersData)) {
 1812 errorMessage = String("Action sequence pointer parameters is not an object");
 1813 return std::nullopt;
 1814 }
 1815
 1816 String pointerType;
 1817 if (!parametersData->getString(ASCIILiteral("pointerType"), pointerType))
 1818 return parameters;
 1819
 1820 if (pointerType == "mouse")
 1821 parameters.pointerType = PointerType::Mouse;
 1822 else if (pointerType == "pen")
 1823 parameters.pointerType = PointerType::Pen;
 1824 else if (pointerType == "touch")
 1825 parameters.pointerType = PointerType::Touch;
 1826 else {
 1827 errorMessage = String("The parameter 'pointerType' in action sequence pointer parameters is invalid");
 1828 return std::nullopt;
 1829 }
 1830
 1831 return parameters;
 1832}
 1833
 1834static std::optional<Vector<Action>> processInputActionSequence(Session& session, JSON::Value& actionSequenceValue, std::optional<String>& errorMessage)
 1835{
 1836 RefPtr<JSON::Object> actionSequence;
 1837 if (!actionSequenceValue.asObject(actionSequence)) {
 1838 errorMessage = String("The action sequence is not an object");
 1839 return std::nullopt;
 1840 }
 1841
 1842 String type;
 1843 actionSequence->getString(ASCIILiteral("type"), type);
 1844 InputSource::Type inputSourceType;
 1845 if (type == "key")
 1846 inputSourceType = InputSource::Type::Key;
 1847 else if (type == "pointer")
 1848 inputSourceType = InputSource::Type::Pointer;
 1849 else if (type == "none")
 1850 inputSourceType = InputSource::Type::None;
 1851 else {
 1852 errorMessage = String("The parameter 'type' is invalid or missing in action sequence");
 1853 return std::nullopt;
 1854 }
 1855
 1856 String id;
 1857 if (!actionSequence->getString(ASCIILiteral("id"), id)) {
 1858 errorMessage = String("The parameter 'id' is invalid or missing in action sequence");
 1859 return std::nullopt;
 1860 }
 1861
 1862 std::optional<PointerParameters> parameters;
 1863 std::optional<PointerType> pointerType;
 1864 if (inputSourceType == InputSource::Type::Pointer) {
 1865 parameters = processPointerParameters(*actionSequence, errorMessage);
 1866 if (!parameters)
 1867 return std::nullopt;
 1868
 1869 pointerType = parameters->pointerType;
 1870 }
 1871
 1872 auto& inputSource = session.getOrCreateInputSource(id, inputSourceType, pointerType);
 1873 if (inputSource.type != inputSourceType) {
 1874 errorMessage = String("Action sequence type doesn't match input source type");
 1875 return std::nullopt;
 1876 }
 1877
 1878 if (inputSource.type == InputSource::Type::Pointer && inputSource.pointerType != pointerType) {
 1879 errorMessage = String("Action sequence pointer type doesn't match input source pointer type");
 1880 return std::nullopt;
 1881 }
 1882
 1883 RefPtr<JSON::Array> actionItems;
 1884 if (!actionSequence->getArray(ASCIILiteral("actions"), actionItems)) {
 1885 errorMessage = String("The parameter 'actions' is invalid or not present in action sequence");
 1886 return std::nullopt;
 1887 }
 1888
 1889 Vector<Action> actions;
 1890 unsigned actionItemsLength = actionItems->length();
 1891 for (unsigned i = 0; i < actionItemsLength; ++i) {
 1892 auto actionItemValue = actionItems->get(i);
 1893 RefPtr<JSON::Object> actionItem;
 1894 if (!actionItemValue->asObject(actionItem)) {
 1895 errorMessage = String("An action in action sequence is not an object");
 1896 return std::nullopt;
 1897 }
 1898
 1899 std::optional<Action> action;
 1900 if (inputSourceType == InputSource::Type::None)
 1901 action = processNullAction(id, *actionItem, errorMessage);
 1902 else if (inputSourceType == InputSource::Type::Key)
 1903 action = processKeyAction(id, *actionItem, errorMessage);
 1904 else if (inputSourceType == InputSource::Type::Pointer)
 1905 action = processPointerAction(id, parameters.value(), *actionItem, errorMessage);
 1906 if (!action)
 1907 return std::nullopt;
 1908
 1909 actions.append(action.value());
 1910 }
 1911
 1912 return actions;
 1913}
 1914
 1915void WebDriverService::performActions(RefPtr<JSON::Object>&& parameters, Function<void (CommandResult&&)>&& completionHandler)
 1916{
 1917 // §17.5 Perform Actions.
 1918 // https://w3c.github.io/webdriver/webdriver-spec.html#perform-actions
 1919 if (!findSessionOrCompleteWithError(*parameters, completionHandler))
 1920 return;
 1921
 1922 RefPtr<JSON::Array> actionsArray;
 1923 if (!parameters->getArray(ASCIILiteral("actions"), actionsArray)) {
 1924 completionHandler(CommandResult::fail(CommandResult::ErrorCode::InvalidArgument, String("The paramater 'actions' is invalid or not present")));
 1925 return;
 1926 }
 1927
 1928 std::optional<String> errorMessage;
 1929 Vector<Vector<Action>> actionsByTick;
 1930 unsigned actionsArrayLength = actionsArray->length();
 1931 for (unsigned i = 0; i < actionsArrayLength; ++i) {
 1932 auto actionSequence = actionsArray->get(i);
 1933 auto inputSourceActions = processInputActionSequence(*m_session, *actionSequence, errorMessage);
 1934 if (!inputSourceActions) {
 1935 completionHandler(CommandResult::fail(CommandResult::ErrorCode::InvalidArgument, errorMessage.value()));
 1936 return;
 1937 }
 1938 for (unsigned i = 0; i < inputSourceActions->size(); ++i) {
 1939 if (actionsByTick.size() < i + 1)
 1940 actionsByTick.append({ });
 1941 actionsByTick[i].append(inputSourceActions.value()[i]);
 1942 }
 1943 }
 1944
 1945 m_session->performActions(WTFMove(actionsByTick), WTFMove(completionHandler));
 1946}
 1947
 1948void WebDriverService::releaseActions(RefPtr<JSON::Object>&& parameters, Function<void (CommandResult&&)>&& completionHandler)
 1949{
 1950 // §17.5 Release Actions.
 1951 // https://w3c.github.io/webdriver/webdriver-spec.html#release-actions
 1952 if (!findSessionOrCompleteWithError(*parameters, completionHandler))
 1953 return;
 1954
 1955 m_session->releaseActions(WTFMove(completionHandler));
 1956}
 1957
15901958void WebDriverService::dismissAlert(RefPtr<JSON::Object>&& parameters, Function<void (CommandResult&&)>&& completionHandler)
15911959{
15921960 // §18.1 Dismiss Alert.

Source/WebDriver/WebDriverService.h

@@private:
103103 void addCookie(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
104104 void deleteCookie(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
105105 void deleteAllCookies(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
 106 void performActions(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
 107 void releaseActions(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
106108 void dismissAlert(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
107109 void acceptAlert(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);
108110 void getAlertText(RefPtr<JSON::Object>&&, Function<void (CommandResult&&)>&&);

Source/WebKit/ChangeLog

 12018-05-09 Carlos Garcia Campos <cgarcia@igalia.com>
 2
 3 WebDriver: implement advance user interactions
 4 https://bugs.webkit.org/show_bug.cgi?id=174616
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 Handle origin in case of mouse move transitions.
 9
 10 * UIProcess/Automation/Automation.json: Add MouseMoveOrigin enum and pass it as parameter of InputSourceState
 11 together with optional node handle. Also pass the frame handle to performInteractionSequence command to find the
 12 node in the current browsing context.
 13 * UIProcess/Automation/SimulatedInputDispatcher.cpp:
 14 (WebKit::SimulatedInputKeyFrame::keyFrameToResetInputSources): Ensure we reset the location.
 15 (WebKit::SimulatedInputDispatcher::resolveLocation): Helper to resolve destination location based on current
 16 location and mouse move origin.
 17 (WebKit::SimulatedInputDispatcher::transitionInputSourceToState): Use resolveLocation() in mouse transitions.
 18 (WebKit::SimulatedInputDispatcher::run): Receive and save the frame ID.
 19 (WebKit::SimulatedInputDispatcher::finishDispatching): Reset the frame ID.
 20 * UIProcess/Automation/SimulatedInputDispatcher.h:
 21 * UIProcess/Automation/WebAutomationSession.cpp:
 22 (WebKit::WebAutomationSession::computeElementLayout): Use even numbers for the callback ID to not conflict with
 23 viewportInViewCenterPointOfElement() callbacks.
 24 (WebKit::WebAutomationSession::didComputeElementLayout): Handle computeElementLayout() or
 25 viewportInViewCenterPointOfElement() requests by calling the right callback depending on whether the ID is odd
 26 or even number.
 27 (WebKit::WebAutomationSession::viewportInViewCenterPointOfElement): Send ComputeElementLayout message to the
 28 WebProcess using odd numbers for the callback ID to not conflict with computeElementLayout() callbacks.
 29 (WebKit::WebAutomationSession::performInteractionSequence): Handle the mouse origin and element handle.
 30 (WebKit::WebAutomationSession::cancelInteractionSequence): Pass the frame ID to the input dispatcher.
 31 * UIProcess/Automation/WebAutomationSession.h:
 32 * UIProcess/Automation/WebAutomationSessionMacros.h:
 33
1342018-05-08 Sihui Liu <sihui_liu@apple.com>
235
336 Adopt new async _savecookies SPI for keeping networking process active during flushing cookies

Source/WebKit/UIProcess/Automation/Automation.json

273273 { "name": "states", "type": "array", "items": { "$ref": "InputSourceState" }, "optional": true, "description": "A list of new input states for input sources that must transition during this step. Source state transitions that cannot be performed concurrently (i.e., touch gestures) are performed sequentially in the order they are listed." }
274274 ]
275275 },
 276 {
 277 "id": "MouseMoveOrigin",
 278 "type": "string",
 279 "description": "Enumerates different origin types that can be used in mouse move interactions.",
 280 "enum": [
 281 "Viewport",
 282 "Pointer",
 283 "Element"
 284 ]
 285 },
276286 {
277287 "id": "InputSourceState",
278288 "type": "object",

282292 { "name": "pressedCharKey", "type": "string", "optional": true, "description": "For 'keyboard' input sources, specifies a character key that has 'pressed' state. Unmentioned character keys are assumed to have a 'released' state." },
283293 { "name": "pressedVirtualKey", "$ref": "VirtualKey", "optional": true, "description": "For 'keyboard' input sources, specifies a virtual key that has a 'pressed' state. Unmentioned virtual keys are assumed to have a 'released' state." },
284294 { "name": "pressedButton", "$ref": "MouseButton", "optional": true, "description": "For 'mouse' input sources, specifies which mouse button has a 'pressed' state. Unmentioned mouse buttons are assumed to have a 'released' state." },
 295 { "name": "origin", "$ref": "MouseMoveOrigin", "optional": true, "description": "For 'mouse' input sources, specifies the origin type of a mouse move transition. Defaults to 'Viewport' if omitted."},
 296 { "name": "nodeHandle", "$ref": "NodeHandle", "optional": true, "description": "The handle of the element to use as origin when origin type is 'Element'."},
285297 { "name": "location", "$ref": "Point", "optional": true, "description": "For 'mouse' or 'touch' input sources, specifies a location in view coordinates to which the input source should transition. Transitioning to this state may interpolate intemediate input source states to better simulate real user movements and gestures." },
286298 { "name": "duration", "type": "integer", "optional": true, "description": "The minimum number of milliseconds that must elapse while the relevant input source transitions to this state." }
287299 ]

459471 "description": "Perform multiple simulated interactions over time using a list of input sources and a list of steps, where each step specifies a state for each input source at the time that step is performed.",
460472 "parameters": [
461473 { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The browsing context to be interacted with." },
 474 { "name": "frameHandle", "$ref": "FrameHandle", "optional": true, "description": "The handle for the frame in which to search for the elements in case of an 'Element' type MouseMoveOrigin. The main frame is used if this parameter empty string or excluded." },
462475 { "name": "inputSources", "type": "array", "items": { "$ref": "InputSource" }, "description": "All input sources that are used to perform this interaction sequence." },
463476 { "name": "steps", "type": "array", "items": { "$ref": "InteractionStep" }, "description": "A list of steps that are executed in order." }
464477 ],

468481 "name": "cancelInteractionSequence",
469482 "description": "Cancel an active interaction sequence that is currently in progress.",
470483 "parameters": [
471  { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The browsing context to be interacted with." }
 484 { "name": "handle", "$ref": "BrowsingContextHandle", "description": "The browsing context to be interacted with." },
 485 { "name": "frameHandle", "$ref": "FrameHandle", "optional": true, "description": "The handle for the frame passed to performInteractionSequence. The main frame is used if this parameter empty string or excluded." }
472486 ],
473487 "async": true
474488 },

Source/WebKit/UIProcess/Automation/SimulatedInputDispatcher.cpp

@@SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameToResetInputSources(HashS
6666 Vector<SimulatedInputKeyFrame::StateEntry> entries;
6767 entries.reserveCapacity(inputSources.size());
6868
69  for (auto& inputSource : inputSources)
70  entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), SimulatedInputSourceState::emptyState() });
 69 for (auto& inputSource : inputSources) {
 70 auto emptyState = SimulatedInputSourceState::emptyState();
 71 // Ensure we reset the location.
 72 emptyState.location = WebCore::IntPoint();
 73 entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), WTFMove(emptyState) });
 74 }
7175
7276 return SimulatedInputKeyFrame(WTFMove(entries));
7377}

@@void SimulatedInputDispatcher::transitionBetweenKeyFrames(const SimulatedInputKe
174178 transitionToNextInputSourceState();
175179}
176180
177 void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource& inputSource, const SimulatedInputSourceState& newState, AutomationCompletionHandler&& completionHandler)
 181void SimulatedInputDispatcher::resolveLocation(const WebCore::IntPoint& currentLocation, std::optional<WebCore::IntPoint> location, MouseMoveOrigin origin, std::optional<String> nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&& completionHandler)
 182{
 183 if (!location) {
 184 completionHandler(currentLocation, std::nullopt);
 185 return;
 186 }
 187
 188 switch (origin) {
 189 case MouseMoveOrigin::Viewport:
 190 completionHandler(location.value(), std::nullopt);
 191 break;
 192 case MouseMoveOrigin::Pointer: {
 193 WebCore::IntPoint destination(currentLocation);
 194 destination.moveBy(location.value());
 195 completionHandler(destination, std::nullopt);
 196 break;
 197 }
 198 case MouseMoveOrigin::Element: {
 199 m_client.viewportInViewCenterPointOfElement(m_page, m_frameID.value(), nodeHandle.value(), [destination = location.value(), completionHandler = WTFMove(completionHandler)](std::optional<WebCore::IntPoint> inViewCenterPoint, std::optional<AutomationCommandError> error) mutable {
 200 if (error) {
 201 completionHandler(std::nullopt, error);
 202 return;
 203 }
 204
 205 ASSERT(inViewCenterPoint);
 206 destination.moveBy(inViewCenterPoint.value());
 207 completionHandler(destination, std::nullopt);
 208 });
 209 break;
 210 }
 211 }
 212}
 213
 214void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource& inputSource, SimulatedInputSourceState& newState, AutomationCompletionHandler&& completionHandler)
178215{
179216 // Make cases and conditionals more readable by aliasing pre/post states as 'a' and 'b'.
180  SimulatedInputSourceState a = inputSource.state;
181  SimulatedInputSourceState b = newState;
 217 SimulatedInputSourceState& a = inputSource.state;
 218 SimulatedInputSourceState& b = newState;
182219
183220 AutomationCompletionHandler eventDispatchFinished = [&inputSource, &newState, completionHandler = WTFMove(completionHandler)](std::optional<AutomationCommandError> error) {
184221 if (error) {

@@void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource
196233 eventDispatchFinished(std::nullopt);
197234 break;
198235 case SimulatedInputSource::Type::Mouse: {
199  // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
200  if (!a.pressedMouseButton && b.pressedMouseButton)
201  m_client.simulateMouseInteraction(m_page, MouseInteraction::Down, b.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
202  else if (a.pressedMouseButton && !b.pressedMouseButton)
203  m_client.simulateMouseInteraction(m_page, MouseInteraction::Up, a.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
204  else if (a.location != b.location) {
205  // FIXME: This does not interpolate mousemoves per the "perform a pointer move" algorithm (§17.4 Dispatching Actions).
206  m_client.simulateMouseInteraction(m_page, MouseInteraction::Move, b.pressedMouseButton.value_or(MouseButton::NoButton), b.location.value(), WTFMove(eventDispatchFinished));
207  } else
208  eventDispatchFinished(std::nullopt);
 236 resolveLocation(a.location.value_or(WebCore::IntPoint()), b.location, b.origin.value_or(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](std::optional<WebCore::IntPoint> location, std::optional<AutomationCommandError> error) mutable {
 237 if (error) {
 238 eventDispatchFinished(error);
 239 return;
 240 }
 241 RELEASE_ASSERT(location);
 242 b.location = location;
 243 // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
 244 if (!a.pressedMouseButton && b.pressedMouseButton)
 245 m_client.simulateMouseInteraction(m_page, MouseInteraction::Down, b.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
 246 else if (a.pressedMouseButton && !b.pressedMouseButton)
 247 m_client.simulateMouseInteraction(m_page, MouseInteraction::Up, a.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
 248 else if (a.location != b.location) {
 249 // FIXME: This does not interpolate mousemoves per the "perform a pointer move" algorithm (§17.4 Dispatching Actions).
 250 m_client.simulateMouseInteraction(m_page, MouseInteraction::Move, b.pressedMouseButton.value_or(MouseButton::NoButton), b.location.value(), WTFMove(eventDispatchFinished));
 251 } else
 252 eventDispatchFinished(std::nullopt);
 253 });
209254 break;
210255 }
211256 case SimulatedInputSource::Type::Keyboard:

@@void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource
225270 }
226271}
227272
228 void SimulatedInputDispatcher::run(Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&& completionHandler)
 273void SimulatedInputDispatcher::run(uint64_t frameID, Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&& completionHandler)
229274{
230275 ASSERT(!isActive());
231276 if (isActive()) {

@@void SimulatedInputDispatcher::run(Vector<SimulatedInputKeyFrame>&& keyFrames, H
233278 return;
234279 }
235280
 281 m_frameID = frameID;
236282 m_runCompletionHandler = WTFMove(completionHandler);
237283 for (const Ref<SimulatedInputSource>& inputSource : inputSources)
238284 m_inputSources.add(inputSource.copyRef());

@@void SimulatedInputDispatcher::finishDispatching(std::optional<AutomationCommand
261307 m_keyFrameTransitionDurationTimer.stop();
262308
263309 auto finish = std::exchange(m_runCompletionHandler, nullptr);
 310 m_frameID = std::nullopt;
264311 m_keyframes.clear();
265312 m_inputSources.clear();
266313 m_keyframeIndex = 0;

Source/WebKit/UIProcess/Automation/SimulatedInputDispatcher.h

@@namespace Inspector { namespace Protocol { namespace Automation {
3939enum class ErrorMessage;
4040enum class KeyboardInteractionType;
4141enum class MouseInteraction;
 42enum class MouseMoveOrigin;
4243enum class VirtualKey;
4344} } }
4445

@@using VirtualKey = Inspector::Protocol::Automation::VirtualKey;
5455using CharKey = char; // For WebDriver, this only needs to support ASCII characters on 102-key keyboard.
5556using MouseButton = WebMouseEvent::Button;
5657using MouseInteraction = Inspector::Protocol::Automation::MouseInteraction;
 58using MouseMoveOrigin = Inspector::Protocol::Automation::MouseMoveOrigin;
5759
5860struct SimulatedInputSourceState {
5961 std::optional<CharKey> pressedCharKey;
6062 std::optional<VirtualKey> pressedVirtualKey;
6163 std::optional<MouseButton> pressedMouseButton;
 64 std::optional<MouseMoveOrigin> origin;
 65 std::optional<String> nodeHandle;
6266 std::optional<WebCore::IntPoint> location;
6367 std::optional<Seconds> duration;
6468

@@public:
112116 virtual ~Client() { }
113117 virtual void simulateMouseInteraction(WebPageProxy&, MouseInteraction, WebMouseEvent::Button, const WebCore::IntPoint& locationInView, AutomationCompletionHandler&&) = 0;
114118 virtual void simulateKeyboardInteraction(WebPageProxy&, KeyboardInteraction, std::optional<VirtualKey>, std::optional<CharKey>, AutomationCompletionHandler&&) = 0;
 119 virtual void viewportInViewCenterPointOfElement(WebPageProxy&, uint64_t frameID, const String& nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&&) = 0;
115120 };
116121
117122 static Ref<SimulatedInputDispatcher> create(WebPageProxy& page, SimulatedInputDispatcher::Client& client)

@@public:
121126
122127 ~SimulatedInputDispatcher();
123128
124  void run(Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&&);
 129 void run(uint64_t frameID, Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&&);
125130 void cancel();
126131
127132 bool isActive() const;

@@private:
133138 void transitionBetweenKeyFrames(const SimulatedInputKeyFrame&, const SimulatedInputKeyFrame&, AutomationCompletionHandler&&);
134139
135140 void transitionToNextInputSourceState();
136  void transitionInputSourceToState(SimulatedInputSource&, const SimulatedInputSourceState& newState, AutomationCompletionHandler&&);
 141 void transitionInputSourceToState(SimulatedInputSource&, SimulatedInputSourceState& newState, AutomationCompletionHandler&&);
137142 void finishDispatching(std::optional<AutomationCommandError>);
138143
139144 void keyFrameTransitionDurationTimerFired();
140145 bool isKeyFrameTransitionComplete() const;
141146
 147 void resolveLocation(const WebCore::IntPoint& currentLocation, std::optional<WebCore::IntPoint> location, MouseMoveOrigin, std::optional<String> nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&&);
 148
142149 WebPageProxy& m_page;
143150 SimulatedInputDispatcher::Client& m_client;
144151
 152 std::optional<uint64_t> m_frameID;
145153 AutomationCompletionHandler m_runCompletionHandler;
146154 AutomationCompletionHandler m_keyFrameTransitionCompletionHandler;
147155 RunLoop::Timer<SimulatedInputDispatcher> m_keyFrameTransitionDurationTimer;

Source/WebKit/UIProcess/Automation/WebAutomationSession.cpp

 1
12/*
23 * Copyright (C) 2016, 2017 Apple Inc. All rights reserved.
34 *

@@void WebAutomationSession::computeElementLayout(const String& browsingContextHan
977978 if (!coordinateSystem)
978979 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "The parameter 'coordinateSystem' is invalid.");
979980
980  uint64_t callbackID = m_nextComputeElementLayoutCallbackID++;
 981 // Start at 2 and use only even numbers to not conflict with m_nextViewportInViewCenterPointOfElementCallbackID.
 982 uint64_t callbackID = m_nextComputeElementLayoutCallbackID += 2;
981983 m_computeElementLayoutCallbacks.set(callbackID, WTFMove(callback));
982984
983985 bool scrollIntoViewIfNeeded = optionalScrollIntoViewIfNeeded ? *optionalScrollIntoViewIfNeeded : false;

@@void WebAutomationSession::computeElementLayout(const String& browsingContextHan
986988
987989void WebAutomationSession::didComputeElementLayout(uint64_t callbackID, WebCore::IntRect rect, std::optional<WebCore::IntPoint> inViewCenterPoint, bool isObscured, const String& errorType)
988990{
 991 if (callbackID % 2 == 1) {
 992 ASSERT(inViewCenterPoint);
 993 if (auto callback = m_viewportInViewCenterPointOfElementCallbacks.take(callbackID)) {
 994 std::optional<AutomationCommandError> error;
 995 if (!errorType.isEmpty())
 996 error = AUTOMATION_COMMAND_ERROR_WITH_MESSAGE(errorType);
 997 callback(inViewCenterPoint, error);
 998 }
 999 return;
 1000 }
 1001
9891002 auto callback = m_computeElementLayoutCallbacks.take(callbackID);
9901003 if (!callback)
9911004 return;

@@SimulatedInputSource* WebAutomationSession::inputSourceForType(SimulatedInputSou
14031416}
14041417
14051418// SimulatedInputDispatcher::Client API
 1419void WebAutomationSession::viewportInViewCenterPointOfElement(WebPageProxy& page, uint64_t frameID, const String& nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&& completionHandler)
 1420{
 1421 // Start at 3 and use only odd numbers to not conflict with m_nextComputeElementLayoutCallbackID.
 1422 uint64_t callbackID = m_nextViewportInViewCenterPointOfElementCallbackID += 2;
 1423 m_viewportInViewCenterPointOfElementCallbacks.set(callbackID, WTFMove(completionHandler));
 1424
 1425 page.process().send(Messages::WebAutomationSessionProxy::ComputeElementLayout(page.pageID(), frameID, nodeHandle, false, CoordinateSystem::LayoutViewport, callbackID), 0);
 1426}
 1427
14061428void WebAutomationSession::simulateMouseInteraction(WebPageProxy& page, MouseInteraction interaction, WebMouseEvent::Button mouseButton, const WebCore::IntPoint& locationInViewport, CompletionHandler<void(std::optional<AutomationCommandError>)>&& completionHandler)
14071429{
14081430 WebCore::IntPoint locationInView = WebCore::IntPoint(locationInViewport.x(), locationInViewport.y() + page.topContentInset());

@@static SimulatedInputSource::Type simulatedInputSourceTypeFromProtocolSourceType
16631685}
16641686#endif // USE(APPKIT) || PLATFORM(GTK)
16651687
1666 void WebAutomationSession::performInteractionSequence(const String& handle, const JSON::Array& inputSources, const JSON::Array& steps, Ref<WebAutomationSession::PerformInteractionSequenceCallback>&& callback)
 1688void WebAutomationSession::performInteractionSequence(const String& handle, const String* optionalFrameHandle, const JSON::Array& inputSources, const JSON::Array& steps, Ref<WebAutomationSession::PerformInteractionSequenceCallback>&& callback)
16671689{
16681690 // This command implements WebKit support for §17.5 Perform Actions.
16691691

@@void WebAutomationSession::performInteractionSequence(const String& handle, cons
16741696 if (!page)
16751697 ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
16761698
 1699 auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
 1700 if (!frameID)
 1701 ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
 1702
16771703 HashMap<String, Ref<SimulatedInputSource>> sourceIdToInputSourceMap;
16781704 HashMap<SimulatedInputSource::Type, String, WTF::IntHash<SimulatedInputSource::Type>, WTF::StrongEnumHashTraits<SimulatedInputSource::Type>> typeToSourceIdMap;
16791705

@@void WebAutomationSession::performInteractionSequence(const String& handle, cons
17581784 sourceState.pressedMouseButton = protocolMouseButtonToWebMouseEventButton(protocolButton.value_or(Inspector::Protocol::Automation::MouseButton::None));
17591785 }
17601786
 1787 String originString;
 1788 if (stateObject->getString(ASCIILiteral("origin"), originString))
 1789 sourceState.origin = Inspector::Protocol::AutomationHelpers::parseEnumValueFromString<Inspector::Protocol::Automation::MouseMoveOrigin>(originString);
 1790
 1791 if (sourceState.origin && sourceState.origin.value() == Inspector::Protocol::Automation::MouseMoveOrigin::Element) {
 1792 String nodeHandleString;
 1793 if (!stateObject->getString(ASCIILiteral("nodeHandle"), nodeHandleString))
 1794 ASYNC_FAIL_WITH_PREDEFINED_ERROR_AND_DETAILS(InvalidParameter, "Node handle not provided for 'Element' origin");
 1795 sourceState.nodeHandle = nodeHandleString;
 1796 }
 1797
17611798 RefPtr<JSON::Object> locationObject;
17621799 if (stateObject->getObject(ASCIILiteral("location"), locationObject)) {
17631800 int x, y;

@@void WebAutomationSession::performInteractionSequence(const String& handle, cons
17821819 }
17831820
17841821 // Delegate the rest of §17.4 Dispatching Actions to the dispatcher.
1785  inputDispatcher.run(WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
 1822 inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
17861823 if (error)
17871824 callback->sendFailure(error.value().toProtocolString());
17881825 else

@@void WebAutomationSession::performInteractionSequence(const String& handle, cons
17911828#endif // PLATFORM(COCOA) || PLATFORM(GTK)
17921829}
17931830
1794 void WebAutomationSession::cancelInteractionSequence(const String& handle, Ref<CancelInteractionSequenceCallback>&& callback)
 1831void WebAutomationSession::cancelInteractionSequence(const String& handle, const String* optionalFrameHandle, Ref<CancelInteractionSequenceCallback>&& callback)
17951832{
17961833 // This command implements WebKit support for §17.6 Release Actions.
17971834

@@void WebAutomationSession::cancelInteractionSequence(const String& handle, Ref<C
18021839 if (!page)
18031840 ASYNC_FAIL_WITH_PREDEFINED_ERROR(WindowNotFound);
18041841
 1842 auto frameID = webFrameIDForHandle(optionalFrameHandle ? *optionalFrameHandle : emptyString());
 1843 if (!frameID)
 1844 ASYNC_FAIL_WITH_PREDEFINED_ERROR(FrameNotFound);
 1845
18051846 Vector<SimulatedInputKeyFrame> keyFrames({ SimulatedInputKeyFrame::keyFrameToResetInputSources(m_inputSources) });
18061847 SimulatedInputDispatcher& inputDispatcher = inputDispatcherForPage(*page);
18071848 inputDispatcher.cancel();
18081849
1809  inputDispatcher.run(WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
 1850 inputDispatcher.run(frameID.value(), WTFMove(keyFrames), m_inputSources, [protectedThis = makeRef(*this), callback = WTFMove(callback)](std::optional<AutomationCommandError> error) {
18101851 if (error)
18111852 callback->sendFailure(error.value().toProtocolString());
18121853 else

Source/WebKit/UIProcess/Automation/WebAutomationSession.h

@@public:
138138 // SimulatedInputDispatcher::Client API
139139 void simulateMouseInteraction(WebPageProxy&, MouseInteraction, WebMouseEvent::Button, const WebCore::IntPoint& locationInView, AutomationCompletionHandler&&) final;
140140 void simulateKeyboardInteraction(WebPageProxy&, KeyboardInteraction, std::optional<VirtualKey>, std::optional<CharKey>, AutomationCompletionHandler&&) final;
 141 void viewportInViewCenterPointOfElement(WebPageProxy&, uint64_t frameID, const String& nodeHandle, Function<void (std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>&&) final;
141142
142143 // Inspector::AutomationBackendDispatcherHandler API
143144 // NOTE: the set of declarations included in this interface depend on the "platform" property in Automation.json

@@public:
159160 void evaluateJavaScriptFunction(const String& browsingContextHandle, const String* optionalFrameHandle, const String& function, const JSON::Array& arguments, const bool* optionalExpectsImplicitCallbackArgument, const int* optionalCallbackTimeout, Ref<Inspector::AutomationBackendDispatcherHandler::EvaluateJavaScriptFunctionCallback>&&) override;
160161 void performMouseInteraction(const String& handle, const JSON::Object& requestedPosition, const String& mouseButton, const String& mouseInteraction, const JSON::Array& keyModifiers, Ref<PerformMouseInteractionCallback>&&) final;
161162 void performKeyboardInteractions(const String& handle, const JSON::Array& interactions, Ref<PerformKeyboardInteractionsCallback>&&) override;
162  void performInteractionSequence(const String& handle, const JSON::Array& sources, const JSON::Array& steps, Ref<PerformInteractionSequenceCallback>&&) override;
163  void cancelInteractionSequence(const String& handle, Ref<CancelInteractionSequenceCallback>&&) override;
 163 void performInteractionSequence(const String& handle, const String* optionalFrameHandle, const JSON::Array& sources, const JSON::Array& steps, Ref<PerformInteractionSequenceCallback>&&) override;
 164 void cancelInteractionSequence(const String& handle, const String* optionalFrameHandle, Ref<CancelInteractionSequenceCallback>&&) override;
164165 void takeScreenshot(const String& handle, const String* optionalFrameHandle, const String* optionalNodeHandle, const bool* optionalScrollIntoViewIfNeeded, const bool* optionalClipToViewport, Ref<TakeScreenshotCallback>&&) override;
165166 void resolveChildFrameHandle(const String& browsingContextHandle, const String* optionalFrameHandle, const int* optionalOrdinal, const String* optionalName, const String* optionalNodeHandle, Ref<ResolveChildFrameHandleCallback>&&) override;
166167 void resolveParentFrameHandle(const String& browsingContextHandle, const String& frameHandle, Ref<ResolveParentFrameHandleCallback>&&) override;

@@private:
276277 uint64_t m_nextResolveParentFrameCallbackID { 1 };
277278 HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ResolveParentFrameHandleCallback>> m_resolveParentFrameHandleCallbacks;
278279
279  uint64_t m_nextComputeElementLayoutCallbackID { 1 };
 280 // Start at 2 and use only even numbers to not conflict with m_nextViewportInViewCenterPointOfElementCallbackID.
 281 uint64_t m_nextComputeElementLayoutCallbackID { 2 };
280282 HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::ComputeElementLayoutCallback>> m_computeElementLayoutCallbacks;
281283
 284 // Start at 3 and use only odd numbers to not conflict with m_nextComputeElementLayoutCallbackID.
 285 uint64_t m_nextViewportInViewCenterPointOfElementCallbackID { 3 };
 286 HashMap<uint64_t, Function<void(std::optional<WebCore::IntPoint>, std::optional<AutomationCommandError>)>> m_viewportInViewCenterPointOfElementCallbacks;
 287
282288 uint64_t m_nextScreenshotCallbackID { 1 };
283289 HashMap<uint64_t, RefPtr<Inspector::AutomationBackendDispatcherHandler::TakeScreenshotCallback>> m_screenshotCallbacks;
284290

Source/WebKit/UIProcess/Automation/WebAutomationSessionMacros.h

3939#define STRING_FOR_PREDEFINED_ERROR_MESSAGE_AND_DETAILS(errorMessage, detailsString) makeString(Inspector::Protocol::AutomationHelpers::getEnumConstantValue(VALIDATED_ERROR_MESSAGE(errorMessage)), errorNameAndDetailsSeparator, detailsString)
4040
4141#define AUTOMATION_COMMAND_ERROR_WITH_NAME(errorName) AutomationCommandError(Inspector::Protocol::Automation::ErrorMessage::errorName)
 42#define AUTOMATION_COMMAND_ERROR_WITH_MESSAGE(errorString) AutomationCommandError(VALIDATED_ERROR_MESSAGE(errorString))
4243
4344// Convenience macros for filling in the error string of synchronous commands in bailout branches.
4445#define SYNC_FAIL_WITH_PREDEFINED_ERROR(errorName) \

WebDriverTests/ChangeLog

 12018-05-09 Carlos Garcia Campos <cgarcia@igalia.com>
 2
 3 WebDriver: implement advance user interactions
 4 https://bugs.webkit.org/show_bug.cgi?id=174616
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 Update test expectations.
 9
 10 * TestExpectations.json:
 11
1122018-04-25 Carlos Garcia Campos <cgarcia@igalia.com>
213
314 Unreviewed gardening. Update expectations for new tests added in r230953.

WebDriverTests/TestExpectations.json

7878 }
7979 },
8080 "imported/selenium/py/test/selenium/webdriver/common/interactions_tests.py": {
81  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 81 "subtests": {
 82 "testClickingOnFormElements": {
 83 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 84 },
 85 "testSelectingMultipleItems": {
 86 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 87 },
 88 "testSendingKeysToActiveElementWithModifier": {
 89 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 90 }
 91 }
8292 },
8393 "imported/selenium/py/test/selenium/webdriver/common/position_and_size_tests.py": {
8494 "subtests": {

177187 }
178188 },
179189 "imported/selenium/py/test/selenium/webdriver/common/w3c_interaction_tests.py": {
180  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 190 "subtests": {
 191 "testSendingKeysToActiveElementWithModifier": {
 192 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 193 }
 194 }
181195 },
182196 "imported/selenium/py/test/selenium/webdriver/common/window_tests.py": {
183197 "subtests": {

193207 }
194208 },
195209 "imported/w3c/webdriver/tests/actions/key.py": {
196  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 210 "subtests": {
 211 "test_single_printable_key_sends_correct_events[\\xe0-]": {
 212 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 213 },
 214 "test_single_printable_key_sends_correct_events[\\u0416-]": {
 215 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 216 },
 217 "test_single_printable_key_sends_correct_events[\\u2603-]": {
 218 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 219 },
 220 "test_single_printable_key_sends_correct_events[\\uf6c2-]": {
 221 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 222 },
 223 "test_single_emoji_records_correct_key[\\U0001f604]": {
 224 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 225 },
 226 "test_single_emoji_records_correct_key[\\U0001f60d]": {
 227 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 228 },
 229 "test_single_modifier_key_sends_correct_events[\\ue053-OSRight-Meta]": {
 230 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 231 },
 232 "test_single_modifier_key_sends_correct_events[\\ue009-ControlLeft-Control]": {
 233 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 234 },
 235 "test_sequence_of_keydown_printable_keys_sends_events": {
 236 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 237 }
 238 }
197239 },
198240 "imported/w3c/webdriver/tests/actions/key_shortcuts.py": {
199  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 241 "subtests": {
 242 "test_mod_a_and_backspace_deletes_all_text": {
 243 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 244 },
 245 "test_mod_a_mod_c_right_mod_v_pastes_text": {
 246 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 247 },
 248 "test_mod_a_mod_x_deletes_all_text": {
 249 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 250 }
 251 }
200252 },
201253 "imported/w3c/webdriver/tests/actions/modifier_click.py": {
202  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
203  },
204  "imported/w3c/webdriver/tests/actions/mouse.py": {
205  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
206  },
207  "imported/w3c/webdriver/tests/actions/mouse_dblclick.py": {
208  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
209  },
210  "imported/w3c/webdriver/tests/actions/mouse_pause_dblclick.py": {
211  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 254 "subtests": {
 255 "test_many_modifiers_click": {
 256 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 257 }
 258 }
212259 },
213260 "imported/w3c/webdriver/tests/actions/pointer_origin.py": {
214  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 261 "subtests": {
 262 "test_element_larger_than_viewport": {
 263 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 264 }
 265 }
215266 },
216267 "imported/w3c/webdriver/tests/actions/sequence.py": {
217  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 268 "subtests": {
 269 "test_release_char_sequence_sends_keyup_events_in_reverse": {
 270 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 271 }
 272 }
218273 },
219274 "imported/w3c/webdriver/tests/actions/special_keys.py": {
220  "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/174616"}}
 275 "subtests": {
 276 "test_webdriver_special_key_sends_keydown[F12-expected10]": {
 277 "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/184967"}}
 278 },
 279 "test_webdriver_special_key_sends_keydown[F11-expected47]": {
 280 "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/184967"}}
 281 },
 282 "test_webdriver_special_key_sends_keydown[F5-expected55]": {
 283 "expected": {"all": {"status": ["SKIP"], "bug": "webkit.org/b/184967"}}
 284 },
 285 "test_webdriver_special_key_sends_keydown[SHIFT-expected3]": {
 286 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 287 },
 288 "test_webdriver_special_key_sends_keydown[R_ARROWRIGHT-expected4]": {
 289 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 290 },
 291 "test_webdriver_special_key_sends_keydown[PAGE_UP-expected6]": {
 292 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 293 },
 294 "test_webdriver_special_key_sends_keydown[R_PAGEUP-expected7]": {
 295 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 296 },
 297 "test_webdriver_special_key_sends_keydown[META-expected11]": {
 298 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 299 },
 300 "test_webdriver_special_key_sends_keydown[NULL-expected15]": {
 301 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 302 },
 303 "test_webdriver_special_key_sends_keydown[SUBTRACT-expected16]": {
 304 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 305 },
 306 "test_webdriver_special_key_sends_keydown[CONTROL-expected17]": {
 307 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 308 },
 309 "test_webdriver_special_key_sends_keydown[R_META-expected19]": {
 310 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 311 },
 312 "test_webdriver_special_key_sends_keydown[SEMICOLON-expected20]": {
 313 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 314 },
 315 "test_webdriver_special_key_sends_keydown[NUMPAD4-expected22]": {
 316 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 317 },
 318 "test_webdriver_special_key_sends_keydown[R_ALT-expected25]": {
 319 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 320 },
 321 "test_webdriver_special_key_sends_keydown[DECIMAL-expected27]": {
 322 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 323 },
 324 "test_webdriver_special_key_sends_keydown[R_DELETE-expected29]": {
 325 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 326 },
 327 "test_webdriver_special_key_sends_keydown[PAGE_DOWN-expected30]": {
 328 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 329 },
 330 "test_webdriver_special_key_sends_keydown[PAUSE-expected31]": {
 331 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 332 },
 333 "test_webdriver_special_key_sends_keydown[R_ARROWUP-expected34]": {
 334 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 335 },
 336 "test_webdriver_special_key_sends_keydown[CLEAR-expected36]": {
 337 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 338 },
 339 "test_webdriver_special_key_sends_keydown[R_ARROWLEFT-expected37]": {
 340 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 341 },
 342 "test_webdriver_special_key_sends_keydown[EQUALS-expected38]": {
 343 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 344 },
 345 "test_webdriver_special_key_sends_keydown[R_PAGEDOWN-expected39]": {
 346 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 347 },
 348 "test_webdriver_special_key_sends_keydown[ADD-expected40]": {
 349 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 350 },
 351 "test_webdriver_special_key_sends_keydown[NUMPAD1-expected41]": {
 352 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 353 },
 354 "test_webdriver_special_key_sends_keydown[R_INSERT-expected42]": {
 355 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 356 },
 357 "test_webdriver_special_key_sends_keydown[ENTER-expected43]": {
 358 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 359 },
 360 "test_webdriver_special_key_sends_keydown[CANCEL-expected44]": {
 361 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 362 },
 363 "test_webdriver_special_key_sends_keydown[NUMPAD6-expected45]": {
 364 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 365 },
 366 "test_webdriver_special_key_sends_keydown[R_END-expected48]": {
 367 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 368 },
 369 "test_webdriver_special_key_sends_keydown[NUMPAD7-expected49]": {
 370 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 371 },
 372 "test_webdriver_special_key_sends_keydown[NUMPAD2-expected50]": {
 373 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 374 },
 375 "test_webdriver_special_key_sends_keydown[F5-expected55]": {
 376 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 377 },
 378 "test_webdriver_special_key_sends_keydown[F6-expected56]": {
 379 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 380 },
 381 "test_webdriver_special_key_sends_keydown[F6-expected56]": {
 382 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 383 },
 384 "test_webdriver_special_key_sends_keydown[F7-expected57]": {
 385 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 386 },
 387 "test_webdriver_special_key_sends_keydown[F7-expected57]": {
 388 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 389 },
 390 "test_webdriver_special_key_sends_keydown[F8-expected58]": {
 391 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 392 },
 393 "test_webdriver_special_key_sends_keydown[F8-expected58]": {
 394 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 395 },
 396 "test_webdriver_special_key_sends_keydown[F9-expected59]": {
 397 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 398 },
 399 "test_webdriver_special_key_sends_keydown[F9-expected59]": {
 400 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 401 },
 402 "test_webdriver_special_key_sends_keydown[NUMPAD8-expected60]": {
 403 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 404 },
 405 "test_webdriver_special_key_sends_keydown[NUMPAD8-expected60]": {
 406 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 407 },
 408 "test_webdriver_special_key_sends_keydown[NUMPAD5-expected61]": {
 409 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 410 },
 411 "test_webdriver_special_key_sends_keydown[NUMPAD5-expected61]": {
 412 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 413 },
 414 "test_webdriver_special_key_sends_keydown[R_CONTROL-expected62]": {
 415 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 416 },
 417 "test_webdriver_special_key_sends_keydown[R_CONTROL-expected62]": {
 418 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 419 },
 420 "test_webdriver_special_key_sends_keydown[R_HOME-expected63]": {
 421 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 422 },
 423 "test_webdriver_special_key_sends_keydown[R_HOME-expected63]": {
 424 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 425 },
 426 "test_webdriver_special_key_sends_keydown[ZENKAKUHANKAKU-expected64]": {
 427 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 428 },
 429 "test_webdriver_special_key_sends_keydown[ZENKAKUHANKAKU-expected64]": {
 430 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 431 },
 432 "test_webdriver_special_key_sends_keydown[R_SHIFT-expected65]": {
 433 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 434 },
 435 "test_webdriver_special_key_sends_keydown[R_SHIFT-expected65]": {
 436 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 437 },
 438 "test_webdriver_special_key_sends_keydown[SEPARATOR-expected66]": {
 439 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 440 },
 441 "test_webdriver_special_key_sends_keydown[SEPARATOR-expected66]": {
 442 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 443 },
 444 "test_webdriver_special_key_sends_keydown[ALT-expected67]": {
 445 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 446 },
 447 "test_webdriver_special_key_sends_keydown[ALT-expected67]": {
 448 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 449 },
 450 "test_webdriver_special_key_sends_keydown[R_ARROWDOWN-expected68]": {
 451 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 452 },
 453 "test_webdriver_special_key_sends_keydown[R_ARROWDOWN-expected68]": {
 454 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 455 },
 456 "test_webdriver_special_key_sends_keydown[DELETE-expected69]": {
 457 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 458 },
 459 "test_webdriver_special_key_sends_keydown[DELETE-expected69]": {
 460 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 461 },
 462 "test_multiple_codepoint_keys_behave_correctly[f]": {
 463 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 464 },
 465 "test_multiple_codepoint_keys_behave_correctly[f]": {
 466 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 467 },
 468 "test_multiple_codepoint_keys_behave_correctly[\u0ba8\u0bbf]": {
 469 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 470 },
 471 "test_multiple_codepoint_keys_behave_correctly[\u0ba8\u0bbf]": {
 472 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 473 },
 474 "test_multiple_codepoint_keys_behave_correctly[\u1100\u1161\u11a8]": {
 475 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 476 },
 477 "test_multiple_codepoint_keys_behave_correctly[\u1100\u1161\u11a8]": {
 478 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 479 },
 480 "test_invalid_multiple_codepoint_keys_fail[fa]": {
 481 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 482 },
 483 "test_invalid_multiple_codepoint_keys_fail[fa]": {
 484 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 485 },
 486 "test_invalid_multiple_codepoint_keys_fail[\u0ba8\u0bbfb]": {
 487 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 488 },
 489 "test_invalid_multiple_codepoint_keys_fail[\u0ba8\u0bbfb]": {
 490 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 491 },
 492 "test_invalid_multiple_codepoint_keys_fail[\u0ba8\u0bbf\u0ba8]": {
 493 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 494 },
 495 "test_invalid_multiple_codepoint_keys_fail[\u0ba8\u0bbf\u0ba8]": {
 496 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 497 },
 498 "test_invalid_multiple_codepoint_keys_fail[\u1100\u1161\u11a8c]": {
 499 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 500 },
 501 "test_invalid_multiple_codepoint_keys_fail[\u1100\u1161\u11a8c]": {
 502 "expected": {"all": {"status": ["FAIL"], "bug": "webkit.org/b/184967"}}
 503 }
 504 }
221505 },
222506 "imported/w3c/webdriver/tests/contexts/maximize_window.py": {
223507 "subtests": {