<?xml version="1.0" encoding="US-ASCII"?>

<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [

<!-- used v2 -->

<!ENTITY I-D.narten-iana-considerations-rfc2434bis SYSTEM "http://xml.resource.org/public/rfc/bibxml3/reference.I-D.narten-iana-considerations-rfc2434bis.xml">

<!ENTITY RFC0793 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.0793.xml">
<!ENTITY RFC1035 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.1035.xml">
<!ENTITY RFC1546 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.1546.xml">
<!ENTITY RFC1884 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.1884.xml">
<!ENTITY RFC2030 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2030.xml">
<!ENTITY RFC2101 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2101.xml">
<!ENTITY RFC2373 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2373.xml">
<!ENTITY RFC2526 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2526.xml">
<!ENTITY RFC2893 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2893.xml">
<!ENTITY RFC2902 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2902.xml">
<!ENTITY RFC2991 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2991.xml">
<!ENTITY RFC3056 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3056.xml">
<!ENTITY RFC3068 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3068.xml">
<!ENTITY RFC3258 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3258.xml">
<!ENTITY RFC3513 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3513.xml">
<!ENTITY RFC3775 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3775.xml">
<!ENTITY RFC3882 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3882.xml">
<!ENTITY RFC3964 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.3964.xml">
<!ENTITY RFC4033 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4033.xml">
<!ENTITY RFC4085 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4085.xml">
<!ENTITY RFC4213 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4213.xml">
<!ENTITY RFC4291 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4291.xml">
<!ENTITY RFC4330 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4330.xml">
<!ENTITY RFC4339 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4339.xml">
<!ENTITY RFC4610 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4610.xml">
<!ENTITY RFC4732 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4732.xml">
<!ENTITY RFC4786 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4786.xml">
<!ENTITY RFC4778 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4778.xml">
<!ENTITY RFC4861 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4861.xml">
<!ENTITY RFC4862 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4862.xml">
<!ENTITY RFC4892 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4892.xml">
<!ENTITY RFC4924 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4924.xml">
<!ENTITY RFC4942 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4942.xml">
<!ENTITY RFC5001 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.5001.xml">
<!ENTITY RFC5905 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.5905.xml">
<!ENTITY RFC5909 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.5909.xml">
<!ENTITY RFC6071 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.6071.xml">
<!ENTITY RFC6382 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.6382.xml">
<!ENTITY RFC6275 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.6275.xml">
<!ENTITY RFC6480 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.6480.xml">
]>
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>

<?rfc rfcedstyle="yes" ?>
<?rfc toc="yes"?>
<?rfc tocdepth="4"?>
<?rfc symrefs="yes"?>
<?rfc sortrefs="yes" ?>
<?rfc compact="yes" ?>
<?rfc subcompact="no" ?>
<rfc category="info" number="7094" ipr="trust200902" submissionType="IAB" consensus="yes">


  <!-- ***** FRONT MATTER ***** -->

  <front>


    <title abbrev="Arch Considerations of IP Anycast">Architectural Considerations of IP Anycast</title>

    <author fullname="Danny McPherson" initials="D.M."
            surname="McPherson">
      <organization>Verisign, Inc.</organization>
      <address>
        <postal>
          <street>12061 Bluemont Way</street>
          <city>Reston</city>
          <region>VA</region>
          <code></code>
          <country>USA</country>
        </postal>
        <email>dmcpherson@verisign.com</email>
      </address>
    </author>

    <author fullname="Dave Oran" initials="D.O."
            surname="Oran">
      <organization>Cisco Systems</organization>
      <address>
        <postal>
          <street></street>
          <country>USA</country>
        </postal>
        <email>oran@cisco.com</email>
      </address>
    </author>

    <author fullname="Dave Thaler" initials="D.T."
            surname="Thaler">
      <organization>Microsoft Corporation</organization>
      <address>
        <postal>
          <street>One Microsoft Way</street>
          <city>Redmond</city>
          <region>WA</region>
          <country>USA</country>
        </postal>
        <email>dthaler@microsoft.com</email>
      </address>
    </author>

    <author fullname="Eric Osterweil" initials="E.O."
            surname="Osterweil">
      <organization>Verisign, Inc.</organization>
      <address>
        <postal>
          <street>12061 Bluemont Way</street>
          <city>Reston</city>
          <region>VA</region>
          <code></code>
          <country>USA</country>
        </postal>
        <email>eosterweil@verisign.com</email>
      </address>
    </author>

    <date month="January" year="2014" />



    <area>General</area>

    <workgroup>Internet Engineering Task Force</workgroup>



    <keyword>anycast</keyword>
    <keyword>architecture</keyword>


    <abstract>
      <t>This memo discusses architectural implications of IP anycast and provides some historical analysis of anycast use by various IETF
        protocols.</t>
    </abstract>
  </front>

  <middle>
    <section title="Overview">
      <t>
        IP anycast is a technique with a long legacy and interesting engineering
        challenges.  However, at its core, it is a relatively simple
        concept.  As described in <xref target="RFC4786">BCP 126</xref>, 
        the general form of IP anycast is the practice of making a particular
        Service Address available in multiple, discrete, autonomous
        locations, such that datagrams sent are routed to one of several
        available locations.
      </t>

      <t>IP anycast is used for at least one critical Internet service: that
         of the Domain Name System <xref target="RFC1035"/>
         root servers.  By late 2007,
         at least 10 of the 13 root name servers were already using IP anycast
         <xref target="RSSAC29"/>.  Use of IP anycast is growing for other applications as
         well.  It has been deployed for over a decade for DNS resolution
         services and is currently used by several DNS Top Level Domain (TLD)
         operators.  IP anycast is also used for other services in operational
         environments, including Network Time Protocol (NTP) <xref target="RFC5905"/>
         services.
      </t>

      <t>
        Anycast addresses are syntactically indistinguishable from unicast
        addresses.  

        Anycast addressing is 

   equivalent to that of unicast in
   multiple locations.  Destination-based routing does best-effort
   delivery of a packet to one interface among the set of interfaces
   asserting reachability for the address.  The expectation of
   delivery is to the "closest" instance as determined by unicast
   routing topology metric(s), and there is also a possibility that
   various load-balancing techniques (e.g., per-packet, per-microflow)
   may be used among multiple equal-cost routes to distribute load for
   an anycasted prefix.

      </t>

      <t>
        Unlike IP unicast, it is not considered an error to assert the same
        anycast address on multiple interfaces within the same or multiple
        systems.
      </t>

      <t>

        When IP anycast is employed, many pitfalls and subtleties exist with 
        applications and transports as well as for routing configuration 
        and operation.  In this document, we aim to capture many of the
        architectural implications of IP anycast.
      </t>

      <t>
        <xref target="RFC4786">BCP 126</xref> discusses several different deployment models with IP
        anycast.  Two additional distinctions beyond that document involve
        "off-link anycast" and "on-link anycast".  "Off-link anycast" takes
        advantage of routing protocol preferences and the IP hop-by-hop
        destination-based forwarding paradigm in order to direct packets to
        the "closest" destination.  This is the traditional method of anycast
        largely considered in <xref target="RFC4786">BCP 126</xref> and can be used for IPv4 and IPv6.
        "On-link anycast" is the formal support of anycast in the address
        resolution (duplicate address detection) protocol and is only standardized for IPv6, with the
        introduction of designated anycast addresses on the anycasted hosts,
        and the Override flag in Neighbor Discovery (ND) Neighbor
        Advertisements (NAs) <xref target="RFC4861"/>. 
        There is no standardized mechanism for this in IPv4.
      </t>

    </section>

    <section title="Background">
      <t>
        As of this writing, the term "anycast" appears in 176 RFCs and 144
        active Internet-Drafts.  The following sections capture some of the
        key appearances and discussion of anycasting within the IETF over the
        years.
      </t>

      <section title="Anycast History">
        <t>
          The first formal specification of anycast was provided in "Host
          Anycasting Service" <xref target="RFC1546"/>.  The authors of this document did a
          good job of capturing most of the issues that exist with IP anycast
          today.
        </t>

        <t>
          One of the first documented uses of anycast was in 1994 for a "Video
          Registry" experiment <xref target="IMR9401"/>.  In the experiment, a UDP query was
          transmitted to an anycasted address to locate the topologically
          closest "supposedly equivalent network resource":
          <list hangIndent="10" style="empty">
            <t>
           A video resource (for example, a catalog server that lists
           available video clips) sends an anycast UDP datagram to locate the
           nearest video registry.  At most one registry responds with a
           unicast UDP datagram containing the registry's IP address.  Said
           resource then opens a TCP connection to that [the received registry address] address and sends a
           request to register itself.  Every 5 minutes or so, each registry
           multicasts to all other registries all of the resources it knows
           from local registration requests.  It also immediately announces
           newly registered resources.  Remotely registered resources not
           heard about for 20 minutes are dropped.
            </t>
          </list>
        </t>

        <t>
          There is also discussion that ISPs began using anycast for DNS
          resolution services around the same time, although no public
          references to support this are available.
        </t>

        <t>
          In 1997, the IAB clarified that IPv4 anycast addresses were pure
          "locators" and could never serve as "identifiers" of hosts or 
          interfaces <xref target="RFC2101"/>.
        </t>

        <t>
          In 1998, the IAB conducted a routing workshop <xref target="RFC2902"/>.  Of the
          conclusions and output action items from the report, an Anycast
          section is contained in Section 2.10.3.  Specifically called out
          is the need to describe the advantages and disadvantages of
          anycast and the belief that local-scoped well-known anycast
          addresses will be useful to some applications.  In the subsequent
          section, an action item was outlined that suggested a BOF should be
          held to plan work on anycast, and if a working group forms, a paper
          on the advantages and the disadvantages of anycast should be included
          as part of the charter.
        </t>

        <t>
          As a result of the recommendation in <xref target="RFC2902"/>,
          an Anycast BOF <xref target="ANYCASTBOF"/> was held at IETF 46 in November of 1999.  A number of uses
          for anycast were discussed.  No firm conclusion was reached regarding
          use of TCP with anycasted services. However, it was observed that
          anycasting was useful for DNS, although it did introduce some new
          complexities.  The use of global anycast was not expected to scale
          (see <xref target="sec:reg_wide_anycast_use"/> below for more discussion)
          and, hence, was expected to be limited to a small number of key uses.
        </t>

        <t>
          In 2001, the Multicast and Anycast Group Membership <xref target="MAGMA"/> WG was
          chartered to address host-to-router signaling, including initial authentication and access control
          issues for multicast and anycast group membership, but other aspects of
          anycast, including architecture and routing, were outside the group's
          scope.
        </t>

        <t>
          Simple Network Time Protocol (SNTP) Version 4 <xref target="RFC2030"/> defined how to use SNTP anycast for server discovery.
          This was extended in <xref target="RFC4330"/> as an NTP-specific "manycast"
          service, in which anycast was used for the discovery part.
        </t>

        <!-- TODO Refs below -->

        <t>
          IPv6 defined some reserved subnet anycast addresses <xref target="RFC2526"/> and
          assigned one to "Mobile IPv6 Home-Agents" <xref target="RFC3775"/> (obsoleted by 
          <xref target="RFC6275"/>).
        </t>

        <t>
          The original IPv6 transition mechanism <xref target="RFC2893"/> made use of IPv4
          anycast addresses as tunnel endpoints for IPv6 encapsulated in IPv4,
          but this was later removed <xref target="RFC4213"/>.  The 6to4 tunneling protocol
          <xref target="RFC3056"/> was augmented by a 6to4 relay anycast prefix 
          <xref target="RFC3068"/> in a move aimed at simplifying the configuration of 6to4 routers.  Incidentally,
          6to4 deployment has shown a fair number of operational and security
          issues <xref target="RFC3964"/> that result from using anycast as a discovery
          mechanism.  Specifically, one inference is that operational
          consideration is needed to ensure that anycast addresses get
          advertised and/or filtered in a way that produces the intended scope
          (e.g., only advertise a route for your 6to4 relay to Autonomous
          Systems (ASes) that conform to your own acceptable usage policy), an attribute that can
          easily become quite operationally expensive.
        </t>

        <t>
          In 2002, DNS' use of anycast was first specified in "Distributing Authoritative
          Name Servers via Shared Unicast Addresses" <xref target="RFC3258"/>.  It is notable
          that it used the term "shared unicast address" rather than "anycast
          address" for the service.
          This distinction was made due to the IPv6 differentiation in the on-link model. "Shared 
          unicast" addresses are unicast (not multicast) in the IPv6 model and, therefore, 
          support the off-link anycast model (described earlier) but not the on-link anycast
          model.  
          At the same time, site-local-scoped well-known addresses began being used for recursive 
          resolvers <xref target="DNS-DISC"/>, but this use was never standardized
          (see below in <xref target="sec:service_disc"/> for more discussion).
        </t>

        <t>
          Anycast was used for routing to rendezvous points (RPs) for 
          PIM <xref target="RFC4610"/>.
        </t>

        <t>
          "Operation of Anycast Services" <xref target="RFC4786">BCP 126</xref> deals with how the routing
          system interacts with anycast services and the operation of anycast
          services. 
        </t>

        <t>
          "Requirements for a Mechanism Identifying a Name Server Instance"
          <xref target="RFC4892"/> cites the use of anycast with DNS as a motivation to
          identify individual name server instances, and the Name Server ID (NSID) option was
          defined for this purpose <xref target="RFC5001"/>.  One could view the addition of
          NSID as an incarnation of locator and identifier separation (where the anycast address
          is a locator and the NSID is an identifier).
        </t>

        <t>
          The IAB's
          "Reflections on Internet Transparency" <xref target="RFC4924"/> briefly mentions
          how violating transparency can also damage global services that use
          anycast.
        </t>
      </section>

      <section title="Anycast in IPv6">
        <t>
          Originally, the IPv6 addressing architecture <xref target="RFC1884"/> 
          <xref target="RFC2373"/>
          <xref target="RFC3513"/> severely restricted the use of anycast addresses.  In
          particular, the architecture provided that anycast addresses must not be used as source addresses and
must not be assigned to IPv6 hosts (i.e., only routers).  These restrictions were later lifted in 2006 <xref target="RFC4291"/>.
        </t>

        <t>
          In fact, the more recent "IPv6 Transition/Co-existence Security
          Considerations" <xref target="RFC4942"/> overview now recommends:
          <list hangIndent="10" style="empty">
            <t>
              To avoid exposing knowledge about the internal structure of
              the network, it is recommended that anycast servers now take
              advantage of the ability to return responses with the anycast
              address as the source address if possible.
            </t>
          </list>
        </t>

        <t>
          As discussed in the Overview, "on-link anycast" is employed expressly
          in IPv6 via ND NAs; see Section 7.2.7 of <xref target="RFC4861"/> for additional
          information.
        </t>
      </section>

      <section anchor="sec:dns_anycast" title="DNS Anycast">
        <t>
          "Distributed Authoritative Name Servers via Shared Unicast Addresses"
          <xref target="RFC3258"/> described how to reach authoritative name servers using
          multiple unicast addresses, each one configured on a different set of servers.
          It stated in Section 2.3:
          <list hangIndent="10" style="empty">
            <t>
              This document presumes that the usual DNS failover methods are the
              only ones used to ensure reachability of the data for clients.  It
              does not advise that the routes be withdrawn in the case of
              failure; it advises instead that the DNS process shutdown so that
              servers on other addresses are queried.  This recommendation
              reflects a choice between performance and operational complexity.
              While it would be possible to have some process withdraw the route
              for a specific server instance when it is not available, there is
              considerable operational complexity involved in ensuring that this
              occurs reliably.  Given the existing DNS failover methods, the
              marginal improvement in performance will not be sufficient to
              justify the additional complexity for most uses.
            </t>
          </list>
          </t>
          
         <t>
          In anycast more generally, most anycast benefits cannot be realized without
          route withdrawals, since traffic will continue to be directed to the link with 
          the failed server. When multiple unicast addresses are used with different sets 
          of servers, a client can still fail over to using a different server address
          and, hence, a different set of servers. There can still be reliability problems,
          however, when each set contains a failed server. 
          If all servers in the same set are on
          the same subnet, such problems could be minimized where address
          resolution within the subnet will cause traffic to go to an available
          server.
        </t>

        <t>
          Other assertions included:
          <list style="symbols">
              <t>It asserted (as an advantage) that no routing changes were needed.</t>

              <t>It recommended stopping DNS processes rather than withdrawing
              routes to deal with failures, data synchronization issues, and
              failover, as provided in the quoted text above.  The spirit of this
              advice was that DNS resolvers may (indeed) reach out and query unavailable DNS
              name servers, but as their queries time out, they will elect to pin themselves
              to other server addresses and, hence, different servers.</t>

              <t>It argued that failure modes involving state were not serious,
                because:
                <list style="symbols">
                    <t>the vast majority of DNS queries are UDP</t>

                    <t>large routing metric disparity among authoritative server
                    instances would localize queries to a single instance for
                    most clients</t>

                    <t>when the resolver tries TCP and it breaks, the resolver
                      will try to move to a different server address.  In order
                      to ensure that this is possible,
                      it is important that the DNS zone be configured with multiple
                      server addresses for different sets of name servers.
                      The advice given in Section 3.3 of 
                      <xref target="DNS-DISC"/>
                      describes, in more detail, why using multiple addresses is
                      important.

                    </t>
                </list>
              </t>
          </list>
        </t>

        <t>
          "Unique Per-Node Origin ASNs for Globally Anycasted Services" 
          <xref target="RFC6382"/> makes recommendations regarding the use of per-node unique
          origin Autonomous System Numbers (ASNs) for globally anycasted critical infrastructure services
          in order to provide routing system discriminators for a given
          anycasted prefix.  The object was to allow network management and
          monitoring techniques, or other operational mechanisms to employ this
          new origin AS as a discriminator in whatever manner fits their
          operating environment, either for detection or policy associated with
          a given anycasted node.
        </t>
      </section>

      <section title="BCP 126 on Operation of Anycast Services">

        <t>
          "Operation of Anycast Services" <xref target="RFC4786">BCP 126</xref> was a product of the IETF's
          GROW working group.  The primary design constraint considered was
          that routing "be stable" for significantly longer than a "transaction
          time", where "transaction time" is loosely defined as "a single
          interaction between a single client and a single server".  It takes
          no position on what applications are suitable candidates for anycast
          usage.
        </t>

        <t>
          Furthermore, it views anycast service disruptions as an operational
          problem: "Operators should be aware that, especially for long running
          flows, there are potential failure modes using anycast that are more
          complex than a simple 'destination unreachable' failure using
          unicast".
        </t>

        <t>
          The document primarily deals with global Internet-wide services
          provided by anycast.  Where internal topology issues are discussed,
          they're mostly regarding routing implications rather than
          application design implications.  BCP 126 also views networks
          employing per&nbhy;packet load balancing on equal cost paths as
          "pathological".  This was also discussed in  <xref target="RFC2991"/>.
        </t>
      </section>
    </section>

    <section title="Principles">
      <section title="Layering and Resiliency">
        <t>
          Preserving the integrity of a modular layered design for IP protocols
          on the Internet is critical to its continued success and flexibility.
          One such consideration is that of whether an application should have
          to adapt to changes in the routing system.
        </t>

        <t>

          Applications should make minimal assumptions
          about routing stability, just as they should make minimal assumptions
          about congestion and packet loss.  When designing applications, it
          would perhaps be safe to assume that the routing system may deliver
          each anycast packet to a different service instance, in any pattern, with
          temporal reordering being a not-so-rare phenomenon.
        </t>

        <t>
          Most stateful transport protocols (e.g., TCP), without modification,
          do not understand the properties of anycast; hence, they will fail
          probabilistically, but possibly catastrophically, when using anycast
          addresses in the presence of "normal" routing dynamics.
          Specifically, if datagrams associated with a given active transaction
          are routed to a new anycasted end system and that end system lacks
          state data associated with the active transaction, 
          the session will be reset; hence, it will need to be reinitiated.
          As another example, different networks have different routing 
          properties and therefore will experience problems under different 
          conditions.  This can lead to a protocol working fine in, say, 
          a test lab but not in the global Internet.
        </t>
      </section>

      <section title="Anycast Addresses as Destinations">
        <t>
          When an anycast address is used as a destination address,
          different packets with the
          same destination IP address may reach different destination hosts,
          even if the packets are generated by the same source host.
          Anycast addresses are thus "safe" to use as destination addresses
          for an application if the following design points are all met:
            <list style="symbols">
              <t>A request message or "one shot" message is self-contained in a
                single transport packet.</t>

              <t>A stateless transport (e.g., UDP) is used for the above.</t>

              <t>Replies are always sent to a unicast address; these can be
                 multipacket since the unicast destination is presumed to be
                 associated with a single "stable" end system and not an
                 anycasted source address. Note that this constrains the use of
                 anycast as source addresses in request messages, since reply
                 messages sent back to that address may reach a device that was
                 not the source that initially triggered it.
               </t>

               <t>The server side of the application keeps no hard state across requests.</t>

               <t>Retries are idempotent; in addition to not assuming server state,
                  they do not encode any assumptions about loss of requests versus
                  loss of replies.</t>
            </list>
        </t>

        <t>
          It is noteworthy, though, that even under the above circumstances ICMP messages 
          against packets with anycast source addresses may be routed to servers other than
          those expected. In addition, Path Maximum Transmission Unit Discovery (PMTUD) 
          can encounter complications when employed against anycast addresses, since
          iterations in the PMTU discovery process may have packets routed to different
          anycast service instances.
        </t>

      </section>

      <section title="Anycast Addresses as Sources">
        <t>
          When an anycast address is used as a source address,
          the source address does not
          uniquely identify the source host; hence, replies might be sent
          to a different host. As noted earlier, this concept is sometimes
          referred to (e.g., in <xref target="RFC3258"/>) as a 
          "shared unicast address".
          Anycast addresses are "safe" to use as source addresses for an application if
          all of the following design points are met:
          <list style="symbols">
              <t>No response message is generated by the receiver
                 with the anycast source used as a destination unless the
                 application has some private state synchronization that allows
                 for the response message arriving at a different instance.
              </t>

              <t>
                The source anycast address is reachable via the interface address
                if unicast reverse path forwarding (RPF) <xref target="RFC4778"/> checking is
                on, or the service address is explicitly provisioned to bypass
                RPF checks. In addition to the application defined in <xref target="RFC4778"/>,
                Section 4.4.5 of <xref target="RFC4786">BCP 126</xref> gives explicit consideration to RPF
                checks in anycasting operations.
              </t>
          </list>
        </t>
      </section>

      <section anchor="sec:service_disc" title="Service Discovery">
        <t>
          Applications able to tolerate an extra round-trip time (RTT) to learn
          a unicast destination address for multipacket exchanges might safely
          use anycast destination addresses for service instance discovery. For
          example, "instance discovery" messages are sent to an anycast
          destination address, and a reply is subsequently sent from the unique
          unicast source address of the interface that received the discovery
          message, or a reply is sent from the anycast source address of the
          interface that received the message, containing the unicast address
          to be used to invoke the service.  Only the latter of these  will
          avoid potential NAT binding and stateful firewall issues.
        </t>

        <t>
          <xref target="DNS-DISC"/> discussed several
          options to address the need to configure DNS servers, including
          the use of a "Well-known Anycast Address" for recursive DNS service configuration in clients to ease
          configuration and allow those systems to ship with these well-known
          addresses configured "from the beginning, as, say, factory default".
          The proposal was later dropped, but the analysis was used in
          publishing <xref target="RFC4339"/>.
        </t>
        <t>
   After the final round of revisions to <xref target="DNS-DISC"/>
   was made, <xref target="RFC4339"/> was published with a very similar focus and
   overlapping content.  The difference was that the writing in
   <xref target="RFC4339"/> focused on analysis, while <xref target="DNS-DISC"/>
   covered both the analysis and a specific proposal.  The proposal
   details were removed in what became <xref target="RFC4339"/> although Section 3.3 of
   that RFC still discusses the approach of using a well&nbhy;known anycast
   address in this scenario.  During publication, the IESG requested that
   the following "IESG Note" be contained in the document:

          <list hangIndent="10" style="empty">
            <t>
              This document describes three different approaches for the
              configuration of DNS name resolution server information in IPv6
              hosts.
            </t>

            <t>
              There is not an IETF consensus on which approach is preferred.
              The analysis in this document was developed by the proponents
              for each approach and does not represent an IETF consensus.
            </t>

            <t>
              The 'RA option' and 'Well-known anycast' approaches described in
              this document are not standardized.  Consequently the analysis
              for these approaches might not be completely applicable to any
              specific proposal that might be proposed in the future.
            </t>
          </list>
        </t>
      </section>
    </section>

    <section title="Analysis">
      <section anchor="sec:reg_wide_anycast_use" title="Regarding Widespread Anycast Use">
        <t>
          Widespread use of anycast for global Internet-wide services or inter&nbhy;domain services has some scaling challenges.  Similar in ways to
          multicast, each service generates at least one unique route in the
          global BGP routing system.  As a result, additional anycast instances
          result in additional paths for a given prefix, which scales super&nbhy;linearly as a function of denseness of inter-domain interconnection
          within the routing system (i.e., more paths result in more resources,
          more network interconnections result in more paths).
        </t>

        <t>
          This is why the Anycast BOF concluded that "the use of global anycast
          addresses was not expected to scale and hence was expected to be
          limited to a small number of key uses".
        </t>

        <t>
          However, one interesting note is that multiple anycast services can
          share a route if they are all located in a single announced prefix
          and if all the servers of all the services are always collocated.
          If the announced prefix is aggregated differently in different 
          locations though, longest-match routing might result in some 
          anycast locations being unreachable. Hence, extra precaution must 
          be taken when aggregating prefixes used by anycast services.
        </t>

      </section>
      <section anchor="sec:transport_implications" title="Transport Implications">
        <t>
          UDP is the "lingua franca" for anycast today.  Stateful transports
          could be enhanced to be more anycast friendly.  This was anticipated
          in Host Anycasting Services <xref target="RFC1546"/>, specifically:
          <list hangIndent="10" style="empty">
            <t>
               The solution to this problem is to only permit anycast
                addresses as the remote address of a TCP SYN segment
                (without the ACK bit set).  A TCP can then initiate a
                connection to an anycast address.  When the SYN-ACK is
                sent back by the host that received the anycast segment,
                the initiating TCP should replace the anycast address of
                its peer, with the address of the host returning the
                SYN-ACK.  (The initiating TCP can recognize the connection
                for which the SYN-ACK is destined by treating the anycast
                address as a wildcard address, which matches any incoming
                SYN-ACK segment with the correct destination port and
                address and source port, provided the SYN-ACK's full
                address, including source address, does not match another
                connection and the sequence numbers in the SYN-ACK are
                correct.)  This approach ensures that a TCP, after
                receiving the SYN-ACK is always communicating with only
                one host.
            </t>
          </list>
        </t>
        <t>
          The reason for such considerations can be illustrated through an example:
          one operationally observed
          shortcoming of using the Transmission Control Protocol (TCP) <xref target="RFC0793"/>  
          and anycast nodes in DNS is that even during the TCP connection
          establishment, IP control packets from a DNS client may initially be routed to one 
          anycast instance, but subsequent IP packets may be delivered to a different anycast
          instance if (for example) a route has changed.  In such a case, the TCP connection
          will likely elicit a connection reset but will certainly result in the disruption of
          the connection.  
        </t>
        <t>
          Multi-address transports (e.g., SCTP) might be more amenable to such
          extensions than TCP.
        </t>

        <t>
          The features needed for address discovery when doing multihoming
          in the transport layer are similar to those needed to support anycast.
        </t>

      </section>

      <section title="Stateful Firewalls, Middleboxes, and Anycast">
        <t>
          Middleboxes (e.g., NATs) and stateful firewalls cause problems
          when used in conjunction with some ways to use anycast.  In particular, a server-side
          transition from an anycast source IP address to a unique unicast
          address may require new or additional session state, and this may
          not exist in the middlebox, as discussed previously in 
          <xref target="sec:service_disc"/>.
        </t>



      </section>

    <section anchor="Security" title="Security Considerations">
      <t>
        Anycast is often deployed to mitigate or at least localize the
        effects of distributed denial-of-service (DDoS) attacks.  For
        example, with the Netgear NTP fiasco <xref target="RFC4085"/> anycast was used in a
        distributed sinkhole model <xref target="RFC3882"/> to mitigate the effects of
        embedded globally routed Internet addresses in network elements.
      </t>

      <t>
        "Internet Denial-of-Service Considerations" <xref target="RFC4732"/> notes that: "A
        number of the root nameservers have since been replicated using
        anycast to further improve their resistance to DoS".
      </t>

      <t>
        "Operation of Anycast Services" <xref target="RFC4786">BCP 126</xref>
        cites DoS mitigation,
        constraining DoS to localized regions, and identifying attack sources
        using spoofed addresses as some motivations to deploy services using
        anycast.  Multiple anycast service instances such as those used by
        the root name servers also add resiliency when network partitioning
        occurs (e.g., as the result of transoceanic fiber cuts or natural
        disasters).
      </t>

      <t> When using anycast, care must be taken not to simply withdraw an
          anycast route in the presence of a sustained DoS attack, since
          the result would simply move the attack to another service
          instance, potentially causing a cascaded failure. Anycast adds
          resiliency when such an attack is instead constrained to a single
          service instance.
      </t>

      <t>
        It should be noted that there is a significant man-in-the-middle
        (MITM) exposure in either variant of anycast discovery (see 
        <xref target="sec:service_disc"/>)
        that, in many applications, may necessitate
        the need for end-to-end security models (e.g., using IPsec <xref target="RFC6071"/> or even DNSSEC <xref target="RFC4033"/>) that enable end systems to authenticate 
        one another, or the data itself.
      </t>

      <t>
        However, when considering the above suggestion of enabling end 
        systems to authenticate each other, a potential complication can arise.
         If the service nodes of an anycast deployment are
  administered by separate authorities,
  any server-side authentication credentials that are used
  must (necessarily) be shared across the administrative boundaries in
  the anycast deployment.  This
        would likely also be the case with Secure Neighbor Discovery, described
        in <xref target="RFC5909"/>.
      </t>

      <t>
        Furthermore, as discussed earlier in this document, operational
        consideration needs to be given to ensure that anycast addresses get
        advertised and/or filtered in a way that produces intended scope (for
        example, only advertise a route to your 6to4 relay to ASes that
        conform to your own Acceptable Use Policy (AUP)). This seems to be operationally expensive,
        and is often vulnerable to errors outside of the local routing
        domain, in particular when anycasted services are deployed with the
        intent to scope associated announcements within some local or
        regional boundary.
      </t>

      <t>
        As previously discussed, <xref target="RFC6382"/> makes recommendations regarding
        the use of per-node unique origin ASNs for globally anycasted
        critical infrastructure services in order to provide routing system
        discriminators for a given anycasted prefix.  Network management and
        monitoring techniques, or other operational mechanisms, may then
        employ this new discriminator in whatever manner fits their operating
        environment, for either detection or policy associated with a given
        anycasted node.
      </t>

      <t>
        Moreover, the use of per-node unique origin ASNs has the additional benefit
        of overcoming complications that might arise with the potential deployment
        of the Resource Public Key Infrastructure (RPKI) <xref target="RFC6480"/>.
        Without per-node unique origin ASNs, the cryptographic certificates needed 
        to attest to the Route Origin Authorizations (ROAs)
        of a multi-administrative deployment of anycast would
        need to be shared.  However, if each service instance has a separate ASN, then
        those ASNs can be managed separately in the RPKI.
      </t>

      <t>
        Unlike multicast (but like unicast), anycast allows traffic stealing.
        That is, with multicast, joining a multicast group doesn't prevent
        anyone else who was receiving the traffic from continuing to receive
        the traffic.  With anycast, adding an anycasted node to the routing
        system can prevent a previous recipient from continuing to receive
        traffic because it may now be delivered to the new node instead.  As
        such, if an unauthorized anycast node can inject a route into the 
        network, or be resolved using ARP/Neighbor Discovery on a link with an 
        authorized anycast node, traffic can be diverted thereby triggering 
        DoS or other attacks.
        Section 6.3 of <xref target="RFC4786">BCP 126</xref> provides expanded discussion on "Service
        Hijacking" and "traffic stealing", and <xref target="FanInfocom13"/> discusses 
        measured instances of anycast nodes and "benign masquerading or
        hostile hijacking of anycast services", by unauthorized nodes.
      </t>

      <t>
        Unlike unicast (but like multicast), the desire is to allow 
        applications to cause route injection. In multicast, one often 
        allows arbitrary applications on hosts to join multicast groups,
        resulting in multicast routing state. Trying to apply that same 
        model to anycast would 
        present new security concerns, which is why <xref target="MAGMA"/> only 
        got so far. The security concerns include:
      </t>

      <t>
        <list style="numbers">
          <t>
            Allowing route injection can cause DOS to a legitimate address
            owner.
          </t>

          <t>
            Allowing route injection consumes routing resources and can
            hence cause DOS to the routing system and impact legitimate
            communications as a result.
          </t>
        </list>
      </t>

      <t>
        These are two of the core issues that were part of the discussion
        during <xref target="RFC1884"/>, the <xref target="ANYCASTBOF"/>, 
        and the <xref target="MAGMA">MAGMA</xref> chartering.
      </t>

      <t>
        Additional security considerations are scattered throughout the list
        of references provided herein.
      </t>

    </section>

    <section title="Deployment Considerations">
      <t>
        <xref target="RFC4786">BCP 126</xref> provides some very solid guidance related to operations of
        anycasted services and, in particular, the operations of DNS.
      </t>



      <t>
        This document covers issues associated with the architectural
        implications of anycast. This document does not address, in any depth,
the fact that there are deployed services with TCP transport using
anycast today.  Evidence exists to suggest that such practice is not "safe" in
        the traditional and architectural sense (as described in 
        <xref target="sec:transport_implications"/>).  
        These sorts of issues are indeed relative, and we recognize sometimes
        unpredictability in the routing system beyond the local
        administrative domain can be manageable.  That is, despite the inherent
        architectural problems in the use of anycast with stateful transport
        and connection-oriented protocols, there is expanding deployment
        (e.g., for content distribution networks) and situations exist where
        it may make sense (e.g., such as with service discovery, short-lived transactions, 
        or in cases where dynamically directing traffic to topologically
        optimal service instances is required).

        In general, operators should
        consider the content and references provided herein and evaluate the
        benefits and implications of anycast in their specific environments
        and applications.
      </t>

      <t>
        In addition, (as noted in <xref target="sec:dns_anycast"/>) the 
        issue of whether to
        withdraw anycast routes when there is a service failure is only
        briefly broached in  <xref target="RFC3258"/>.  The advice given is that routes should
        not be withdrawn, in order to reduce operational complexity.
        However, the issue of route advertisements and service outages
        deserves greater attention.
      </t>
      <t>
        There is an inherent trade-off that exists
        between the operational complexity of matching service outages with
        anycast route withdrawals, and allowing anycast routes to persist for
        services that are no longer available.  <xref target="RFC3258"/> maintains that DNS'
        inherent failure recovery mechanism is sufficient to overcome failed
        nodes, but even this advice enshrines the notion that these decisions
        are both application-specific and subject to the operational needs of
        each deployment.  For example, the routing system plays a larger role
        in DNS when services are anycast.  Therefore, operational consideration must
        be given to the fact that relying on anycast for DNS deployment
        optimizations means that there are operational trade-offs related to
        keeping route advertisements (and withdrawals) symmetric with service
        availability.  For example, in order to ensure that the DNS resolvers in a
        failed anycast instance's catchment <xref target="RFC4786"/> are able to fail over and
        reach a non-failed catchment, a route withdrawal is almost certainly
        required.  On the other hand, instability of a DNS process that
        triggers frequent route advertisement and withdrawal might result in
        suppression of legitimate paths to available nodes, e.g., as a result of route
        flap damping <xref target="RFC2439"/>.
      </t>

      <t>
        Rather than prescribing advice that attempts to befit all situations,
        it should simply be recognized that when using anycast with network
        services that provide redundancy or resilience capabilities at other
        layers of the protocol stack, operators should carefully consider the
        optimal layer(s) at which to provide said functions.
      </t>
      <t>
        As noted in <xref target="sec:dns_anycast"/>, use of anycast within a subnet does not
   necessarily suffer from the potential issues with route withdrawals.
   As such, use of anycast to reach servers that reside in the same
   subnet can be made more reliable than use of anycast to reach
   topologically disparate server instances.  Within a subnet, however,
   care must be taken as stated in Section 5.4 of <xref target="RFC4862"/>, "Duplicate
   Address Detection MUST NOT be performed on anycast addresses";
   hence, the servers must be configured appropriately.
      </t>

    </section>
  </section>

  <section title="Conclusions">
    <t>
      In summary, operators and application vendors alike should consider
      the benefits and implications of anycast in their specific
      environments and applications and also give forward consideration to
      how new network protocols and application functions may take
      advantage of anycast or how they may be negatively impacted if
      anycasting is employed.
    </t>
  </section>

  <section anchor="Acknowledgements" title="Acknowledgements">
    <t>
      Many thanks to Kurtis Lindqvist for his early review and
      feedback on this document.  Thanks to Brian Carpenter, Alfred
      Hoenes, and Joe Abley for their usual careful review and
      feedback, as well as Mark Smith, Lixia Zhang, Stephane
      Bortzmeyer, Masataka Ohta, and S. Moonesamy for their detailed
      reviews.  Helpful feedback was also received from others
      including Edward Lewis, Jean-Michel Combes, Wolfgang Nagele,
      Mark Townsley, and Abdussalam Baryun.
    </t>
  </section>


  </middle>

  <!--  *****BACK MATTER ***** -->

  <back>


    <references title="Informative References">
      <reference anchor="ANYCASTBOF" target="http://www.ietf.org/mail-archive/web/ietf/current/msg11182.html">
        <front>
          <title abbrev="ANYCAST BOF">IAB Anycast BOF Announcement</title>
          <author initials="S" surname="Deering">
            <organization></organization>
          </author>
          <date month="October" year="1999"/>
        </front>
      </reference>

      <reference anchor="FanInfocom13">
        <front>
          <title>Evaluating Anycast in the Domain Name System</title>
          <author initials="X" surname="Fan">
            <organization></organization>
          </author>
          <author initials="J" surname="Heidemann">
            <organization></organization>
          </author>
          <author initials="R" surname="Govindan">
            <organization></organization>
          </author>
          <date month="April" year="2013"/>
        </front>
        <seriesInfo name="Proceedings of the IEEE Infocom" value="2013"/>
      </reference>

      <reference anchor="IMR9401" target="ftp://ftp.rfc&nbhy;editor.org/in-notes/museum/imr/imr9401.txt">
        <front>
          <title>INTERNET MONTHLY REPORT</title>
          <author>
             <organization>RFC Editor</organization>
             </author>
          <date month="January" year="1994"/>
        </front>
      </reference>

      <reference anchor="MAGMA" target="http://www.ietf.org/wg/concluded/magma">
        <front>
          <title>Multicast and Anycast Group Membership (MAGMA)</title>
          <author>
            <organization>MAGMA (concluded)</organization>
           </author>
          <date month="April" year="2006"/>
          <workgroup>Multicast and Anycast Group Membership (MAGMA)</workgroup>
          <keyword>concluded</keyword>
          <keyword>IETF</keyword>
          <keyword>http://www.ietf.org/wg/concluded/magma</keyword>
        </front>
      </reference>

      <reference anchor="RSSAC29" target="http://www.icann.org/en/groups/rssac/meetings/rssac-29-en.pdf">
        <front>
          <title>RSSAC 29 Meeting Minutes</title>
          <author/>
          <date day="2" month="December" year="2007"/>
        </front>
      </reference>

      <reference anchor="RFC2439">
        <front>
          <title>BGP Route Flap Damping</title>
          <author initials="C." surname="Villamizar" fullname="Curtis Villamizar">
            <organization>ANS</organization>
            <address>
              <email>curtis@ans.net</email>
            </address>
            </author>
            <author initials="R." surname="Chandra" fullname="Ravi Chandra">
              <organization>Cisco Systems</organization>
              <address>
                <email>rchandra@cisco.com</email>
              </address>
              </author>
              <author initials="R." surname="Govindan" fullname="Ramesh Govindan">
                <organization>ISI</organization>
                <address>
                  <email>govindan@isi.edu</email>
                </address>
              </author>
              <date year="1998" month="November"/>
              <area>Routing</area>
            </front>
            <seriesInfo name="RFC" value="2439"/>
          </reference>

          <!-- &I-D.ietf-ipv6-dns-discovery EXPIRED; -->
      
      <reference anchor="DNS-DISC">
        <front>
          <title>Well known site local unicast addresses for DNS resolver</title>
          <author initials="A" surname="Durand" fullname="Alain Durand">
            <organization/>
          </author>
          <author initials="J" surname="Hagino" fullname="Jun-ichiro Hagino">
            <organization/>
          </author>
          <author initials="D" surname="Thaler" fullname="Dave Thaler">
            <organization/>
          </author>
          <date month="September" day="19" year="2002"/>
        </front>
         <seriesInfo name="Work in" value="Progress" />
      </reference>

      &RFC0793;

      &RFC1035;

      &RFC1546;

      &RFC1884;

      &RFC2030;

      &RFC2101;

      &RFC2373;

      &RFC2526;

      &RFC2893;

      &RFC2902;

      &RFC2991;

      &RFC3056;

      &RFC3068;

      &RFC3258;

      &RFC3513;

      &RFC3775;

      &RFC3882;

      &RFC3964;

      &RFC4033;

      &RFC4085;

      &RFC4213;

      &RFC4291;

      &RFC4330;

      &RFC4339;

      &RFC4610;

      &RFC4862;

      &RFC4732;

      &RFC4778;

      &RFC4786;

      &RFC4861;

      &RFC4892;

      &RFC4924;

      &RFC4942;

      &RFC5001;

      &RFC5905;

      &RFC5909;

      &RFC6071;

      &RFC6275;

      &RFC6382;

      &RFC6480;


    </references>



    <section title="IAB Members at the Time of Approval">
        <figure>
            <artwork>
Bernard Aboba
Jari Arkko
Marc Blanchet
Ross Callon
Alissa Cooper
Joel Halpern
Russ Housley
Eliot Lear
Xing Li
Erik Nordmark
Andrew Sullivan
Dave Thaler
Hannes Tschofenig
            </artwork>
        </figure>
    </section>

  </back>
</rfc>
